From 44c07c30ad2b39a528b9e3aa5aaa4ea1bc07088f Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Sun, 26 Jan 2014 23:56:09 +0100
Subject: [PATCH 01/41] add couch_mrview:view_changes_since/{6,7} function

This function add the possibility to get changes in a view since the
last upddated sequence. You can all changes in a view since a sequence
or all changes for a key or a range in a view.

The following new secondaries are created to allows this feature:

- a generic log index to log the latest changes in views for a docid :
{DocId, [{ViewId, {Key, Seq, OP}}]} where OP can be del or add. This
index allows us to mark a key as removed if needed. It will be useful
later to help us to chain map/reduces operations or such things.

- a seq index associated to a view id : {ViewId, [{Seq, Key}, {DocId, Val}]} to look for all changes in a view

- an index indexing keys by seq: {ViewId, [{[Key, Seq], DocId}, Val}]},
  to looks for changes associated to a key or a ranhge

Note: all deleted keys are marked as deleted in the log index and their
value is {[{<<"_removed">>, true}]}.

To start to index changes you need to pass the options {seq_indexed:
true} to the design document.

Caveat: when the changes are indexed the size of the index is significantly higher.

Example of usage:  https://www.friendpaste.com/5Y6gihQReaxd8ERqbDom3y

Conflicts:
	src/couch_mrview_updater.erl
	src/couch_mrview_util.erl
---
 include/couch_mrview.hrl     |   7 +-
 src/couch_mrview.erl         |  48 +++++++++
 src/couch_mrview_index.erl   |   7 +-
 src/couch_mrview_updater.erl | 184 +++++++++++++++++++++++++++-------
 src/couch_mrview_util.erl    | 186 +++++++++++++++++++++++++++++------
 5 files changed, 364 insertions(+), 68 deletions(-)

diff --git a/include/couch_mrview.hrl b/include/couch_mrview.hrl
index 36c35d6..f1911db 100644
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@ -18,12 +18,13 @@
     idx_name,
     language,
     design_opts=[],
+    seq_indexed=false,
     lib,
     views,
     id_btree=nil,
+    log_btree=nil,
     update_seq=0,
     purge_seq=0,
-
     first_build,
     partial_resp_pid,
     doc_acc,
@@ -41,6 +42,9 @@
     reduce_funs=[],
     def,
     btree=nil,
+    seq_btree=nil,
+    key_byseq_btree=nil,
+    seq_indexed=false,
     options=[]
 }).
 
@@ -49,6 +53,7 @@
     seq=0,
     purge_seq=0,
     id_btree_state=nil,
+    log_btree_state=nil,
     view_states=nil
 }).
 
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 047bc00..afb6f1e 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -15,6 +15,7 @@
 -export([validate/2]).
 -export([query_all_docs/2, query_all_docs/4]).
 -export([query_view/3, query_view/4, query_view/6]).
+-export([view_changes_since/6, view_changes_since/7]).
 -export([get_info/2]).
 -export([trigger_update/2, trigger_update/3]).
 -export([compact/2, compact/3, cancel_compaction/2]).
@@ -135,6 +136,30 @@ query_view(Db, {Type, View, Ref}, Args, Callback, Acc) ->
         erlang:demonitor(Ref, [flush])
     end.
 
+view_changes_since(Db, DDoc, VName, StartSeq, Fun, Acc) ->
+    view_changes_since(Db, DDoc, VName, StartSeq, Fun, [], Acc).
+
+view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
+    Args0 = make_view_changes_args(Options),
+    {ok, {_, View}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+                                                          Args0),
+    case View#mrview.seq_indexed of
+        true ->
+            OptList = make_view_changes_opts(StartSeq, Options, Args),
+            Btree = case is_key_byseq(Options) of
+                true -> View#mrview.key_byseq_btree;
+                _ -> View#mrview.seq_btree
+            end,
+            io:format("opt list ~p~n", [OptList]),
+            AccOut = lists:foldl(fun(Opts, Acc0) ->
+                        {ok, _R, A} = couch_mrview_util:fold_changes(
+                                    Btree, Fun, Acc0, Opts),
+                        A
+                end, Acc, OptList),
+            {ok, AccOut};
+        _ ->
+            {error, seqs_not_indexed}
+    end.
 
 get_info(Db, DDocId) when is_binary(DDocId) ->
     DbName = mem3:dbname(Db#db.name),
@@ -447,3 +472,26 @@ lookup_index(Key) ->
         record_info(fields, mrargs), lists:seq(2, record_info(size, mrargs))
     ),
     couch_util:get_value(Key, Index).
+
+
+is_key_byseq(Options) ->
+    lists:any(fun({K, _}) ->
+                lists:member(K, [start_key, end_key, start_key_docid,
+                                 end_key_docid, keys])
+        end, Options).
+
+make_view_changes_args(Options) ->
+    case is_key_byseq(Options) of
+        true ->
+            to_mrargs(Options);
+        false ->
+            #mrargs{}
+    end.
+
+make_view_changes_opts(StartSeq, Options, Args) ->
+    case is_key_byseq(Options) of
+        true ->
+            couch_mrview_util:changes_key_opts(StartSeq, Args);
+        false ->
+            [[{start_key, {StartSeq+1, <<>>}}] ++ Options]
+    end.
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 2d6ccae..0b9b236 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -51,14 +51,17 @@ get(Property, State) ->
             #mrst{
                 fd = Fd,
                 sig = Sig,
-                id_btree = Btree,
+                id_btree = IdBtree,
+                log_btree = LogBtree,
                 language = Lang,
                 update_seq = UpdateSeq,
                 purge_seq = PurgeSeq,
                 views = Views
             } = State,
             {ok, FileSize} = couch_file:bytes(Fd),
-            {ok, ExternalSize} = couch_mrview_util:calculate_external_size(Views),
+            {ok, ExternalSize} = couch_mrview_util:calculate_external_size(IdBtree,
+                                                                           LogBtree,
+                                                                           Views),
             ActiveSize = ExternalSize + couch_btree:size(Btree),
             {ok, [
                 {signature, list_to_binary(couch_index_util:hexsig(Sig))},
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 1525a4c..eb06c92 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -141,12 +141,12 @@ map_docs(Parent, State0) ->
                 ({nil, Seq, _}, {SeqAcc, Results}) ->
                     {erlang:max(Seq, SeqAcc), Results};
                 ({Id, Seq, deleted}, {SeqAcc, Results}) ->
-                    {erlang:max(Seq, SeqAcc), [{Id, []} | Results]};
+                    {erlang:max(Seq, SeqAcc), [{Id, Seq, []} | Results]};
                 ({Id, Seq, Doc}, {SeqAcc, Results}) ->
                     couch_stats:increment_counter([couchdb, mrview, map_docs],
                                                   1),
                     {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
-                    {erlang:max(Seq, SeqAcc), [{Id, Res} | Results]}
+                    {erlang:max(Seq, SeqAcc), [{Id, Seq, Res} | Results]}
             end,
             FoldFun = fun(Docs, Acc) ->
                 update_task(length(Docs)),
@@ -162,8 +162,8 @@ write_results(Parent, State) ->
     case accumulate_writes(State, State#mrst.write_queue, nil) of
         stop ->
             Parent ! {new_state, State};
-        {Go, {Seq, ViewKVs, DocIdKeys}} ->
-            NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys),
+        {Go, {Seq, ViewKVs, DocIdKeys, Log}} ->
+            NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys, Log),
             if Go == stop ->
                 Parent ! {new_state, NewState};
             true ->
@@ -185,17 +185,17 @@ start_query_server(State) ->
 
 
 accumulate_writes(State, W, Acc0) ->
-    {Seq, ViewKVs, DocIdKVs} = case Acc0 of
-        nil -> {0, [{V#mrview.id_num, []} || V <- State#mrst.views], []};
+    {Seq, ViewKVs, DocIdKVs, Log} = case Acc0 of
+        nil -> {0, [{V#mrview.id_num, {[], []}} || V <- State#mrst.views], [], dict:new()};
         _ -> Acc0
     end,
     case couch_work_queue:dequeue(W) of
         closed when Seq == 0 ->
             stop;
         closed ->
-            {stop, {Seq, ViewKVs, DocIdKVs}};
+            {stop, {Seq, ViewKVs, DocIdKVs, Log}};
         {ok, Info} ->
-            {_, _, NewIds} = Acc = merge_results(Info, Seq, ViewKVs, DocIdKVs),
+            {_, _, NewIds, _} = Acc = merge_results(Info, Seq, ViewKVs, DocIdKVs, Log),
             case accumulate_more(length(NewIds)) of
                 true -> accumulate_writes(State, W, Acc);
                 false -> {ok, Acc}
@@ -212,66 +212,100 @@ accumulate_more(NumDocIds) ->
         andalso CurrMem < list_to_integer(MinSize).
 
 
-merge_results([], SeqAcc, ViewKVs, DocIdKeys) ->
-    {SeqAcc, ViewKVs, DocIdKeys};
-merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys) ->
-    Fun = fun(RawResults, {VKV, DIK}) ->
-        merge_results(RawResults, VKV, DIK)
+merge_results([], SeqAcc, ViewKVs, DocIdKeys, Log) ->
+    {SeqAcc, ViewKVs, DocIdKeys, Log};
+merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Log) ->
+    Fun = fun(RawResults, {VKV, DIK, Log2}) ->
+        merge_results(RawResults, VKV, DIK, Log2)
     end,
-    {ViewKVs1, DocIdKeys1} = lists:foldl(Fun, {ViewKVs, DocIdKeys}, Results),
-    merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1).
+    {ViewKVs1, DocIdKeys1, Log1} = lists:foldl(Fun, {ViewKVs, DocIdKeys, Log},
+                                               Results),
+    merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1,
+                  Log1).
 
 
-merge_results({DocId, []}, ViewKVs, DocIdKeys) ->
-    {ViewKVs, [{DocId, []} | DocIdKeys]};
-merge_results({DocId, RawResults}, ViewKVs, DocIdKeys) ->
+merge_results({DocId, _Seq, []}, ViewKVs, DocIdKeys, Log) ->
+    {ViewKVs, [{DocId, []} | DocIdKeys], dict:store(DocId, [], Log)};
+merge_results({DocId, Seq, RawResults}, ViewKVs, DocIdKeys, Log) ->
     JsonResults = couch_query_servers:raw_to_ejson(RawResults),
     Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
-    {ViewKVs1, ViewIdKeys} = insert_results(DocId, Results, ViewKVs, [], []),
-    {ViewKVs1, [ViewIdKeys | DocIdKeys]}.
+    {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Results, ViewKVs, [],
+                                            [], Log),
+    {ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}.
 
 
-insert_results(DocId, [], [], ViewKVs, ViewIdKeys) ->
-    {lists:reverse(ViewKVs), {DocId, ViewIdKeys}};
-insert_results(DocId, [KVs | RKVs], [{Id, VKVs} | RVKVs], VKVAcc, VIdKeys) ->
+insert_results(DocId, _Seq, [], [], ViewKVs, ViewIdKeys, Log) ->
+    {lists:reverse(ViewKVs), {DocId, ViewIdKeys}, Log};
+insert_results(DocId, Seq, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVAcc,
+               VIdKeys, Log) ->
     CombineDupesFun = fun
-        ({Key, Val}, {[{Key, {dups, Vals}} | Rest], IdKeys}) ->
-            {[{Key, {dups, [Val | Vals]}} | Rest], IdKeys};
-        ({Key, Val1}, {[{Key, Val2} | Rest], IdKeys}) ->
-            {[{Key, {dups, [Val1, Val2]}} | Rest], IdKeys};
-        ({Key, _}=KV, {Rest, IdKeys}) ->
-            {[KV | Rest], [{Id, Key} | IdKeys]}
+        ({Key, Val}, {[{Key, {dups, Vals}} | Rest], IdKeys, Log2}) ->
+            {[{Key, {dups, [Val | Vals]}} | Rest], IdKeys, Log2};
+        ({Key, Val1}, {[{Key, Val2} | Rest], IdKeys, Log2}) ->
+            {[{Key, {dups, [Val1, Val2]}} | Rest], IdKeys, Log2};
+        ({Key, _}=KV, {Rest, IdKeys, Log2}) ->
+            {[KV | Rest], [{Id, Key} | IdKeys],
+             dict:append(DocId, {Id, {Key, Seq, add}}, Log2)}
     end,
-    InitAcc = {[], VIdKeys},
+    InitAcc = {[], VIdKeys, Log},
     couch_stats:increment_counter([couchdb, mrview, emits], length(KVs)),
-    {Duped, VIdKeys0} = lists:foldl(CombineDupesFun, InitAcc, lists:sort(KVs)),
+    {Duped, VIdKeys0, Log1} = lists:foldl(CombineDupesFun, InitAcc,
+                                          lists:sort(KVs)),
     FinalKVs = [{{Key, DocId}, Val} || {Key, Val} <- Duped] ++ VKVs,
-    insert_results(DocId, RKVs, RVKVs, [{Id, FinalKVs} | VKVAcc], VIdKeys0).
+    FinalSKVs = [{{Seq, Key}, {DocId, Val}} || {Key, Val} <- Duped] ++ SKVs,
+    insert_results(DocId, Seq, RKVs, RVKVs,
+                  [{Id, {FinalKVs, FinalSKVs}} | VKVAcc], VIdKeys0, Log1).
 
 
-write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys) ->
+write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
     #mrst{
         id_btree=IdBtree,
+        log_btree=LogBtree,
         first_build=FirstBuild
     } = State,
 
     {ok, ToRemove, IdBtree2} = update_id_btree(IdBtree, DocIdKeys, FirstBuild),
     ToRemByView = collapse_rem_keys(ToRemove, dict:new()),
 
-    UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, KVs}) ->
+    {ok, SeqsToAdd, SeqsToRemove, LogBtree2} = case LogBtree of
+        nil -> {ok, undefined, undefined, nil};
+        _ -> update_log(LogBtree, Log, UpdateSeq, FirstBuild)
+    end,
+
+    UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs, SKVs}}) ->
         ToRem = couch_util:dict_find(ViewId, ToRemByView, []),
         {ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, KVs, ToRem),
         NewUpdateSeq = case VBtree2 =/= View#mrview.btree of
             true -> UpdateSeq;
             _ -> View#mrview.update_seq
         end,
-        View#mrview{btree=VBtree2, update_seq=NewUpdateSeq}
+
+        %% store the view changes.
+        {SeqBtree2, KeyBySeqBtree2} = case View#mrview.seq_indexed of
+            true ->
+                SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+                SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
+                SKVs1 = SKVs ++ SToAdd,
+                {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
+                                                   SKVs1, SToRem),
+
+                {ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
+                                                    couch_mrview_util:to_key_seq(SKVs1),
+                                                    couch_mrview_util:to_key_seq(SToRem)),
+                {SBt, KSbt};
+            _ -> {nil, nil}
+        end,
+        View#mrview{btree=VBtree2,
+                    seq_btree=SeqBtree2,
+                    key_byseq_btree=KeyBySeqBtree2,
+                    update_seq=NewUpdateSeq}
     end,
 
     State#mrst{
         views=lists:zipwith(UpdateView, State#mrst.views, ViewKVs),
         update_seq=UpdateSeq,
-        id_btree=IdBtree2
+        id_btree=IdBtree2,
+        log_btree=LogBtree2
     }.
 
 
@@ -284,6 +318,84 @@ update_id_btree(Btree, DocIdKeys, _) ->
     ToRem = [Id || {Id, DIKeys} <- DocIdKeys, DIKeys == []],
     couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem).
 
+walk_log(BTree, Fun, Acc, Ids) ->
+    WrapFun = fun(KV, _Offset, Acc2) ->
+            Fun(KV, Acc2)
+    end,
+    lists:foldl(fun(Id, Acc1) ->
+                Opt = [{start_key, Id}, {end_key, Id}],
+                {ok, _, A} = couch_btree:fold(BTree, WrapFun, Acc1, Opt),
+                A
+        end, Acc, Ids).
+
+update_log(Btree, Log, _UpdatedSeq, true) ->
+    ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log),
+                             DIKeys /= []],
+    {ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
+    {ok, dict:new(), dict:new(), LogBtree2};
+update_log(Btree, Log, UpdatedSeq, _) ->
+    %% build list of updated keys and Id
+    {ToLook, Updated} = dict:fold(fun
+                (Id, [], {IdsAcc, KeysAcc}) ->
+                    {[Id | IdsAcc], KeysAcc};
+                (Id, DIKeys, {IdsAcc, KeysAcc}) ->
+                    KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}},
+                                               KeysAcc2) ->
+                                    [{Id, ViewId, Key} | KeysAcc2]
+                            end, KeysAcc, DIKeys),
+                {[Id | IdsAcc], KeysAcc1} end, {[], []}, Log),
+
+    io:format("updated ~p~n", [Updated]),
+    RemValue = {[{<<"_removed">>, true}]},
+    {Log1, AddAcc, DelAcc} = walk_log(Btree, fun({DocId, VIdKeys},
+                                                          {Log2, AddAcc2, DelAcc2}) ->
+
+                {Log3, AddAcc3, DelAcc3} = lists:foldl(fun({ViewId,{Key, Seq, Op}},
+                                                           {Log4, AddAcc4, DelAcc4}) ->
+
+                            case lists:member({DocId, ViewId, Key}, Updated) of
+                                true ->
+                                    %% the log is updated, deleted old
+                                    %% record from the view
+                                    DelAcc5 = dict:append(ViewId, {Seq, Key},
+                                                        DelAcc4),
+                                    {Log4, AddAcc4, DelAcc5};
+                                false when Op /= del ->
+                                    %% an update operation has been
+                                    %% logged for this key. We must now
+                                    %% record it as deleted in the
+                                    %% log, remove the old record in
+                                    %% the view and update the view
+                                    %% with a removed record.
+                                    Log5 = dict:append(DocId,
+                                                       {ViewId,
+                                                        {Key,UpdatedSeq, del}},
+                                                       Log4),
+                                    DelAcc5 = dict:append(ViewId, {Seq, Key},
+                                                        DelAcc4),
+                                    AddAcc5 = dict:append(ViewId,
+                                                          {{UpdatedSeq, Key},
+                                                           {DocId, RemValue}},
+                                                          AddAcc4),
+                                    {Log5, AddAcc5, DelAcc5};
+                                false ->
+                                    %% the key has already been
+                                    %% registered in the view as
+                                    %% deleted, make sure to add it
+                                    %% to the new log.
+                                    Log5 = dict:append(DocId,
+                                                       {ViewId,
+                                                        {Key, Seq, del}}, Log4),
+                                    {Log5, AddAcc4, DelAcc4}
+                            end
+                    end, {Log2, AddAcc2, DelAcc2}, VIdKeys),
+                    {ok, {Log3, AddAcc3, DelAcc3}}
+            end, {Log, dict:new(), dict:new()}, ToLook),
+
+    ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log1), DIKeys /= []],
+    %% store the new logs
+    {ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
+    {ok, AddAcc, DelAcc, LogBtree2}.
 
 collapse_rem_keys([], Acc) ->
     Acc;
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 7fc60f8..cd25887 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -21,13 +21,16 @@
 -export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
 -export([fold/4, fold_reduce/4]).
 -export([temp_view_to_ddoc/1]).
--export([calculate_external_size/1]).
+-export([calculate_external_size/3]).
 -export([validate_args/1]).
 -export([maybe_load_doc/3, maybe_load_doc/4]).
 -export([maybe_update_index_file/1]).
 -export([extract_view/4, extract_view_reduce/1]).
 -export([get_view_keys/1, get_view_queries/1]).
 -export([set_view_type/3]).
+-export([changes_key_opts/2]).
+-export([fold_changes/4]).
+-export([to_key_seq/1]).
 
 -define(MOD, couch_mrview_index).
 
@@ -91,15 +94,18 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
                             [Name, Else]),
             DictBySrcAcc
     end,
+    {DesignOpts} = proplists:get_value(<<"options">>, Fields, {[]}),
+    SeqIndexed = proplists:get_value(<<"seq_indexed">>, DesignOpts, false),
+
     {RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
     BySrc = lists:foldl(MakeDict, dict:new(), RawViews),
 
-    NumViews = fun({_, View}, N) -> {View#mrview{id_num=N}, N+1} end,
+    NumViews = fun({_, View}, N) ->
+            {View#mrview{id_num=N, seq_indexed=SeqIndexed}, N+1}
+    end,
     {Views, _} = lists:mapfoldl(NumViews, 0, lists:sort(dict:to_list(BySrc))),
 
     Language = couch_util:get_value(<<"language">>, Fields, <<"javascript">>),
-    {DesignOpts} = couch_util:get_value(<<"options">>, Fields, {[]}),
-    {RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
     Lib = couch_util:get_value(<<"lib">>, RawViews, {[]}),
 
     IdxState = #mrst{
@@ -108,7 +114,8 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
         lib=Lib,
         views=Views,
         language=Language,
-        design_opts=DesignOpts
+        design_opts=DesignOpts,
+        seq_indexed=SeqIndexed
     },
     SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
     {ok, IdxState#mrst{sig=couch_util:md5(term_to_binary(SigInfo))}}.
@@ -152,7 +159,8 @@ view_sig(Db, State, View, #mrargs{include_docs=true}=Args) ->
     BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}),
     UpdateSeq = couch_db:get_update_seq(Db),
     PurgeSeq = couch_db:get_purge_seq(Db),
-    Bin = term_to_binary({BaseSig, UpdateSeq, PurgeSeq}),
+    Bin = term_to_binary({BaseSig, UpdateSeq, PurgeSeq,
+                          State#mrst.seq_indexed}),
     couch_index_util:hexsig(couch_util:md5(Bin));
 view_sig(Db, State, {_Nth, _Lang, View}, Args) ->
     view_sig(Db, State, View, Args);
@@ -160,11 +168,12 @@ view_sig(_Db, State, View, Args0) ->
     Sig = State#mrst.sig,
     UpdateSeq = View#mrview.update_seq,
     PurgeSeq = View#mrview.purge_seq,
+    SeqIndexed = View#mrview.seq_indexed,
     Args = Args0#mrargs{
         preflight_fun=undefined,
         extra=[]
     },
-    Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, Args}),
+    Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, SeqIndexed, Args}),
     couch_index_util:hexsig(couch_util:md5(Bin)).
 
 
@@ -173,7 +182,8 @@ init_state(Db, Fd, #mrst{views=Views}=State, nil) ->
         seq=0,
         purge_seq=couch_db:get_purge_seq(Db),
         id_btree_state=nil,
-        view_states=[{nil, 0, 0} || _ <- Views]
+        log_btree_state=nil,
+        view_states=[{nil, nil, nil, 0, 0} || _ <- Views]
     },
     init_state(Db, Fd, State, Header);
 % read <= 1.2.x header record and transpile it to >=1.3.x
@@ -187,25 +197,31 @@ init_state(Db, Fd, State, #index_header{
         seq=Seq,
         purge_seq=PurgeSeq,
         id_btree_state=IdBtreeState,
-        view_states=ViewStates
+        log_btree_state=nil,
+        view_states=[{Bt, nil, nil, USeq, PSeq} || {Bt, USeq, PSeq} <- ViewStates]
         });
 init_state(Db, Fd, State, Header) ->
-    #mrst{language=Lang, views=Views} = State,
+    #mrst{language=Lang, views=Views, seq_indexed=SeqIndexed} = State,
     #mrheader{
         seq=Seq,
         purge_seq=PurgeSeq,
         id_btree_state=IdBtreeState,
+        log_btree_state=LogBtreeState,
         view_states=ViewStates
     } = Header,
 
     StateUpdate = fun
-        ({_, _, _}=St) -> St;
-        (St) -> {St, 0, 0}
+        ({_, _, _, _, _}=St) -> St;
+        (St) -> {St, nil, nil, 0, 0}
     end,
     ViewStates2 = lists:map(StateUpdate, ViewStates),
 
     IdBtOpts = [{compression, couch_db:compression(Db)}],
     {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts),
+    {ok, LogBtree} = case SeqIndexed of
+        true -> couch_btree:open(LogBtreeState, Fd, IdBtOpts);
+        false -> {ok, nil}
+    end,
 
     OpenViewFun = fun(St, View) -> open_view(Db, Fd, Lang, St, View) end,
     Views2 = lists:zipwith(OpenViewFun, ViewStates2, Views),
@@ -216,11 +232,20 @@ init_state(Db, Fd, State, Header) ->
         update_seq=Seq,
         purge_seq=PurgeSeq,
         id_btree=IdBtree,
+        log_btree=LogBtree,
         views=Views2
     }.
 
+less_json_seqs({SeqA, JsonA}, {SeqB, JsonB}) ->
+    case couch_ejson_compare:less(SeqA, SeqB) of
+        0 ->
+            couch_ejson_compare:less_json(JsonA, JsonB);
+        Result ->
+            Result < 0
+    end.
+
 
-open_view(Db, Fd, Lang, {BTState, USeq, PSeq}, View) ->
+open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
     FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs],
     ReduceFun =
         fun(reduce, KVs) ->
@@ -245,7 +270,23 @@ open_view(Db, Fd, Lang, {BTState, USeq, PSeq}, View) ->
         {compression, couch_db:compression(Db)}
     ],
     {ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
-    View#mrview{btree=Btree, update_seq=USeq, purge_seq=PSeq}.
+
+    {SeqBtree, KeyBySeqBtree} = case View#mrview.seq_indexed of
+        true ->
+            ViewSeqBtOpts = [{less, fun less_json_seqs/2},
+                             {compression, couch_db:compression(Db)}],
+            {ok, SBt} = couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts),
+            {ok, KSBt} = couch_btree:open(KSeqBTState, Fd, ViewBtOpts),
+            {SBt, KSBt};
+        false ->
+            {nil, nil}
+    end,
+
+    View#mrview{btree=Btree,
+                seq_btree=SeqBtree,
+                key_byseq_btree=KeyBySeqBtree,
+                update_seq=USeq,
+                purge_seq=PSeq}.
 
 
 temp_view_to_ddoc({Props}) ->
@@ -300,7 +341,6 @@ fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
     end,
     {ok, _LastRed, _Acc} = couch_btree:fold(Bt, WrapperFun, Acc, Opts).
 
-
 fold_fun(_Fun, [], _, Acc) ->
     {ok, Acc};
 fold_fun(Fun, [KV|Rest], {KVReds, Reds}, Acc) ->
@@ -311,6 +351,12 @@ fold_fun(Fun, [KV|Rest], {KVReds, Reds}, Acc) ->
             {stop, Acc2}
     end.
 
+fold_changes(Bt, Fun, Acc, Opts) ->
+    WrapperFun = fun(KV, _Reds, Acc2) ->
+        Fun(changes_expand_dups([KV], []), Acc2)
+    end,
+    {ok, _LastRed, _Acc} = couch_btree:fold(Bt, WrapperFun, Acc, Opts).
+
 
 fold_reduce({NthRed, Lang, View}, Fun,  Acc, Options) ->
     #mrview{
@@ -495,21 +541,34 @@ make_header(State) ->
         update_seq=Seq,
         purge_seq=PurgeSeq,
         id_btree=IdBtree,
+        log_btree=LogBtree,
         views=Views
     } = State,
-    ViewStates = [
-        {
-            couch_btree:get_state(V#mrview.btree),
-            V#mrview.update_seq,
-            V#mrview.purge_seq
-        }
-        ||
-        V <- Views
-    ],
+
+    ViewStates = lists:foldr(fun(V, Acc) ->
+                    {SeqBtState, KSeqBtState} = case V#mrview.seq_indexed of
+                        true ->
+                            {couch_btree:get_state(V#mrview.seq_btree),
+                             couch_btree:get_state(V#mrview.key_byseq_btree)};
+                        _ -> {nil, nil}
+                    end,
+                    [{couch_btree:get_state(V#mrview.btree),
+                      SeqBtState,
+                      KSeqBtState,
+                      V#mrview.update_seq,
+                      V#mrview.purge_seq} | Acc]
+            end, [], Views),
+
+    LogBtreeState = case LogBtree of
+        nil -> nil;
+        _ -> couch_btree:get_state(LogBtree)
+    end,
+
     #mrheader{
         seq=Seq,
         purge_seq=PurgeSeq,
         id_btree_state=couch_btree:get_state(IdBtree),
+        log_btree_state= LogBtreeState,
         view_states=ViewStates
     }.
 
@@ -567,7 +626,9 @@ reset_state(State) ->
         qserver=nil,
         update_seq=0,
         id_btree=nil,
-        views=[View#mrview{btree=nil} || View <- State#mrst.views]
+        log_btree=nil,
+        views=[View#mrview{btree=nil, seq_btree=nil, key_byseq_btree=nil}
+               || View <- State#mrst.views]
     }.
 
 
@@ -636,11 +697,62 @@ reverse_key_default(<<255>>) -> <<>>;
 reverse_key_default(Key) -> Key.
 
 
-calculate_external_size(Views) ->
-    SumFun = fun(#mrview{btree=Bt}, Acc) ->
-        sum_btree_sizes(Acc, couch_btree:size(Bt))
+changes_key_opts(StartSeq, Args) ->
+    changes_key_opts(StartSeq, Args, []).
+
+
+changes_key_opts(StartSeq, #mrargs{keys=undefined, direction=Dir}=Args, Extra) ->
+    [[{dir, Dir}] ++ changes_skey_opts(StartSeq, Args) ++
+     changes_ekey_opts(StartSeq, Args) ++ Extra];
+changes_key_opts(StartSeq, #mrargs{keys=Keys, direction=Dir}=Args, Extra) ->
+    lists:map(fun(K) ->
+        [{dir, Dir}]
+        ++ changes_skey_opts(StartSeq, Args#mrargs{start_key=K})
+        ++ changes_ekey_opts(StartSeq, Args#mrargs{end_key=K})
+        ++ Extra
+    end, Keys).
+
+
+changes_skey_opts(StartSeq, #mrargs{start_key=undefined}) ->
+    [{start_key, [<<>>, StartSeq+1]}];
+changes_skey_opts(StartSeq, #mrargs{start_key=SKey,
+                                    start_key_docid=SKeyDocId}) ->
+    [{start_key, {[SKey, StartSeq+1], SKeyDocId}}].
+
+
+changes_ekey_opts(_StartSeq, #mrargs{end_key=undefined}) ->
+    [];
+changes_ekey_opts(_StartSeq, #mrargs{end_key=EKey,
+                                    end_key_docid=EKeyDocId,
+                                    direction=Dir}=Args) ->
+    EndSeq = case Dir of
+        fwd -> 16#10000000;
+        rev -> 0
+    end,
+
+    case Args#mrargs.inclusive_end of
+        true -> [{end_key, {[EKey, EndSeq], EKeyDocId}}];
+        false -> [{end_key_gt, {[EKey, EndSeq], EKeyDocId}}]
+    end.
+
+
+
+calculate_external_size(IdBt, LogBt, Views) ->
+    SumFun = fun
+        (#mrview{btree=Bt, seq_btree=nil}, Acc) ->
+            sum_btree_sizes(Acc, couch_btree:size(Bt));
+        (#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
+            Acc1 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
+            Acc2 = sum_btree_sizes(Acc1, couch_btree:size(SBt)),
+            sum_btree_sizes(Acc2, couch_btree:size(KSBt))
+    end,
+    Size = case LogBt of
+        nil ->
+            lists:foldl(SumFun, couch_btree:size(IdBt), Views);
+        _ ->
+            lists:foldl(SumFun, couch_btree:size(IdBt) +
+                        couch_btree:size(LogBt), Views)
     end,
-    Size = lists:foldl(SumFun, 0, Views),
     {ok, Size}.
 
 
@@ -669,6 +781,19 @@ expand_dups([KV | Rest], Acc) ->
     expand_dups(Rest, [KV | Acc]).
 
 
+changes_expand_dups([], Acc) ->
+    lists:reverse(Acc);
+changes_expand_dups([{{[Key, Seq], DocId}, {dups, Vals}} | Rest], Acc) ->
+    Expanded = [{{Key, Seq, DocId}, Val} || Val <- Vals],
+    changes_expand_dups(Rest, Expanded ++ Acc);
+changes_expand_dups([{{Key, Seq}, {DocId, {dups, Vals}}} | Rest], Acc) ->
+    Expanded = [{{Key, Seq, DocId}, Val} || Val <- Vals],
+    changes_expand_dups(Rest, Expanded ++ Acc);
+changes_expand_dups([{{[Key, Seq], DocId}, Val} | Rest], Acc) ->
+    changes_expand_dups(Rest, [{{Key, Seq, DocId}, Val} | Acc]);
+changes_expand_dups([{{Key, Seq}, {DocId, Val}} | Rest], Acc) ->
+    changes_expand_dups(Rest, [{{Key, Seq, DocId}, Val} | Acc]).
+
 maybe_load_doc(_Db, _DI, #mrargs{include_docs=false}) ->
     [];
 maybe_load_doc(Db, #doc_info{}=DI, #mrargs{conflicts=true, doc_options=Opts}) ->
@@ -718,6 +843,9 @@ mrverror(Mesg) ->
     throw({query_parse_error, Mesg}).
 
 
+to_key_seq(L) ->
+    [{{[Key, Seq], DocId}, Val} || {{Seq, Key}, {DocId, Val}} <- L].
+
 %% Updates 1.2.x or earlier view files to 1.3.x or later view files
 %% transparently, the first time the 1.2.x view file is opened by
 %% 1.3.x or later.

From deb8139b8ce94965abba6beb842b9579b569bd2a Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Mon, 27 Jan 2014 10:14:59 +0100
Subject: [PATCH 02/41] couch_mrview: fix purge when seq_indexed=true

This change makes tsure to also purge the log btree and related btrees
in views when seqs_indexed is set to true.
---
 src/couch_mrview.erl         |  1 -
 src/couch_mrview_updater.erl | 57 ++++++++++++++++++++++++++----------
 2 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index afb6f1e..343b91a 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -150,7 +150,6 @@ view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
                 true -> View#mrview.key_byseq_btree;
                 _ -> View#mrview.seq_btree
             end,
-            io:format("opt list ~p~n", [OptList]),
             AccOut = lists:foldl(fun(Opts, Acc0) ->
                         {ok, _R, A} = couch_mrview_util:fold_changes(
                                     Btree, Fun, Acc0, Opts),
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index eb06c92..27cf795 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -55,40 +55,66 @@ start_update(Partial, State, NumChanges) ->
 purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
     #mrst{
         id_btree=IdBtree,
+        log_btree=LogBtree,
         views=Views
     } = State,
 
     Ids = [Id || {Id, _Revs} <- PurgedIdRevs],
-    {ok, Lookups, IdBtree2} = couch_btree:query_modify(IdBtree, Ids, [], Ids),
+    {ok, Lookups, LLookups, LogBtree2, IdBtree2} = case LogBtree of
+        nil ->
+            {ok, L, Bt} = couch_btree:query_modify(IdBtree, Ids, [], Ids),
+            {ok, L, [], nil, Bt};
+        _ ->
+            {ok, L, Bt} = couch_btree:query_modify(IdBtree, Ids, [], Ids),
+            {ok, LL, LBt} = couch_btree:query_modify(LogBtree, Ids, [], Ids),
+            {ok, L, LL, LBt, Bt}
+    end,
 
     MakeDictFun = fun
         ({ok, {DocId, ViewNumRowKeys}}, DictAcc) ->
-            FoldFun = fun({ViewNum, RowKey}, DictAcc2) ->
-                dict:append(ViewNum, {RowKey, DocId}, DictAcc2)
+            FoldFun = fun
+                ({ViewNum, {Key, Seq, _Op}}, DictAcc2) ->
+                    dict:append(ViewNum, {Key, Seq, DocId}, DictAcc2);
+                ({ViewNum, RowKey}, DictAcc2) ->
+                    dict:append(ViewNum, {RowKey, DocId}, DictAcc2)
             end,
             lists:foldl(FoldFun, DictAcc, ViewNumRowKeys);
         ({not_found, _}, DictAcc) ->
             DictAcc
     end,
     KeysToRemove = lists:foldl(MakeDictFun, dict:new(), Lookups),
+    SeqsToRemove = lists:foldl(MakeDictFun, dict:new(), LLookups),
+
+    RemKeysFun = fun(#mrview{id_num=ViewId}=View) ->
+        ToRem = couch_util:dict_find(ViewId, KeysToRemove, []),
+        {ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, [], ToRem),
+        NewPurgeSeq = case VBtree2 =/= View#mrview.btree of
+            true -> PurgeSeq;
+            _ -> View#mrview.purge_seq
+        end,
+        {SeqBtree2, KeyBySeqBtree2} = case View#mrview.seq_indexed of
+            true ->
+                SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+                SKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+                KSs = [{[Seq, Key], DocId} || {Key, Seq, DocId} <- SToRem],
+                {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
+                                                   [], SKs),
+                {ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
+                                                    [], KSs),
+                {SBt, KSbt};
+            _ -> {nil, nil}
+        end,
+        View#mrview{btree=VBtree2,
+                    seq_btree=SeqBtree2,
+                    key_byseq_btree=KeyBySeqBtree2,
+                    purge_seq=NewPurgeSeq}
 
-    RemKeysFun = fun(#mrview{id_num=Num, btree=Btree}=View) ->
-        case dict:find(Num, KeysToRemove) of
-            {ok, RemKeys} ->
-                {ok, Btree2} = couch_btree:add_remove(Btree, [], RemKeys),
-                NewPurgeSeq = case Btree2 /= Btree of
-                    true -> PurgeSeq;
-                    _ -> View#mrview.purge_seq
-                end,
-                View#mrview{btree=Btree2, purge_seq=NewPurgeSeq};
-            error ->
-                View
-        end
     end,
 
     Views2 = lists:map(RemKeysFun, Views),
     {ok, State#mrst{
         id_btree=IdBtree2,
+        log_btree=LogBtree2,
         views=Views2,
         purge_seq=PurgeSeq
     }}.
@@ -345,7 +371,6 @@ update_log(Btree, Log, UpdatedSeq, _) ->
                             end, KeysAcc, DIKeys),
                 {[Id | IdsAcc], KeysAcc1} end, {[], []}, Log),
 
-    io:format("updated ~p~n", [Updated]),
     RemValue = {[{<<"_removed">>, true}]},
     {Log1, AddAcc, DelAcc} = walk_log(Btree, fun({DocId, VIdKeys},
                                                           {Log2, AddAcc2, DelAcc2}) ->

From 19be43b6bea268e02ef2859a9f7e36eb1c186d4d Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Mon, 27 Jan 2014 10:24:25 +0100
Subject: [PATCH 03/41] couch_mrview: fix changes log

This changes make sure we pass correct keys to remove to the seq and
key_byseq btrees in views.
---
 src/couch_mrview_updater.erl | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 27cf795..f6557cf 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -311,13 +311,15 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
             true ->
                 SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
                 SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
+                RemSKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+                RemKSs = [{[Seq, Key], DocId} || {Key, Seq, DocId} <- SToRem],
                 SKVs1 = SKVs ++ SToAdd,
                 {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
-                                                   SKVs1, SToRem),
+                                                   SKVs1, RemSKs),
 
                 {ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
                                                     couch_mrview_util:to_key_seq(SKVs1),
-                                                    couch_mrview_util:to_key_seq(SToRem)),
+                                                    RemKSs),
                 {SBt, KSbt};
             _ -> {nil, nil}
         end,
@@ -382,8 +384,9 @@ update_log(Btree, Log, UpdatedSeq, _) ->
                                 true ->
                                     %% the log is updated, deleted old
                                     %% record from the view
-                                    DelAcc5 = dict:append(ViewId, {Seq, Key},
-                                                        DelAcc4),
+                                    DelAcc5 = dict:append(ViewId,
+                                                          {Key, Seq, DocId},
+                                                          DelAcc4),
                                     {Log4, AddAcc4, DelAcc5};
                                 false when Op /= del ->
                                     %% an update operation has been
@@ -396,8 +399,9 @@ update_log(Btree, Log, UpdatedSeq, _) ->
                                                        {ViewId,
                                                         {Key,UpdatedSeq, del}},
                                                        Log4),
-                                    DelAcc5 = dict:append(ViewId, {Seq, Key},
-                                                        DelAcc4),
+                                    DelAcc5 = dict:append(ViewId,
+                                                          {Key, Seq, DocId},
+                                                          DelAcc4),
                                     AddAcc5 = dict:append(ViewId,
                                                           {{UpdatedSeq, Key},
                                                            {DocId, RemValue}},

From 8185ca7af804c40f292960d8c3c9687af117c167 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Mon, 27 Jan 2014 11:51:11 +0100
Subject: [PATCH 04/41] couch_mrview: add
 couch_mrview:count_view_changes_since/{4,5}

Add function to couch changes in a view index, similar to
couch_db:count_changes_since/2 .

Conflicts:
	src/couch_mrview.erl
---
 src/couch_mrview.erl      | 33 +++++++++++++++++++++++++++++----
 src/couch_mrview_util.erl |  8 ++++++--
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 343b91a..2f68fc0 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -16,6 +16,7 @@
 -export([query_all_docs/2, query_all_docs/4]).
 -export([query_view/3, query_view/4, query_view/6]).
 -export([view_changes_since/6, view_changes_since/7]).
+-export([count_view_changes_since/4, count_view_changes_since/5]).
 -export([get_info/2]).
 -export([trigger_update/2, trigger_update/3]).
 -export([compact/2, compact/3, cancel_compaction/2]).
@@ -160,10 +161,34 @@ view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
             {error, seqs_not_indexed}
     end.
 
-get_info(Db, DDocId) when is_binary(DDocId) ->
-    DbName = mem3:dbname(Db#db.name),
-    {ok, DDoc} = ddoc_cache:open(DbName, DDocId),
-    get_info(Db, DDoc);
+count_view_changes_since(Db, DDoc, VName, SinceSeq) ->
+    count_view_changes_since(Db, DDoc, VName, SinceSeq, []).
+
+count_view_changes_since(Db, DDoc, VName, SinceSeq, Options) ->
+    Args0 = make_view_changes_args(Options),
+    {ok, {_, View}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+                                                          Args0),
+    case View#mrview.seq_indexed of
+        true ->
+            OptList = make_view_changes_opts(SinceSeq, Options, Args),
+            Btree = case is_key_byseq(Options) of
+                true -> View#mrview.key_byseq_btree;
+                _ -> View#mrview.seq_btree
+            end,
+            lists:foldl(fun(Opts, Acc0) ->
+                            {ok, N} = couch_btree:fold_reduce(
+                                    Btree, fun(_SeqStart, PartialReds, 0) ->
+                                        {ok, couch_btree:final_reduce(
+                                                    Btree, PartialReds)}
+                                    end,
+                                0, Opts),
+                            Acc0 + N
+                    end, 0, OptList);
+        _ ->
+            {error, seqs_not_indexed}
+    end.
+
+
 get_info(Db, DDoc) ->
     {ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
     couch_index:get_info(Pid).
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index cd25887..6d48ed0 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -244,7 +244,6 @@ less_json_seqs({SeqA, JsonA}, {SeqB, JsonB}) ->
             Result < 0
     end.
 
-
 open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
     FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs],
     ReduceFun =
@@ -273,10 +272,15 @@ open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
 
     {SeqBtree, KeyBySeqBtree} = case View#mrview.seq_indexed of
         true ->
+            BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
             ViewSeqBtOpts = [{less, fun less_json_seqs/2},
+                             {reduce, BySeqReduceFun},
                              {compression, couch_db:compression(Db)}],
+            KeyBySeqBtOpts = [{less, Less},
+                              {reduce, BySeqReduceFun},
+                              {compression, couch_db:compression(Db)}],
             {ok, SBt} = couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts),
-            {ok, KSBt} = couch_btree:open(KSeqBTState, Fd, ViewBtOpts),
+            {ok, KSBt} = couch_btree:open(KSeqBTState, Fd, KeyBySeqBtOpts),
             {SBt, KSBt};
         false ->
             {nil, nil}

From fb341eafdf877842bbf536d31a7a53f55a6f6345 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Mon, 27 Jan 2014 17:24:54 +0100
Subject: [PATCH 05/41] couch_mrview: add view changes test

test view changes and fix errors.
---
 src/couch_mrview_test_util.erl |  31 +++++++--
 src/couch_mrview_updater.erl   |   2 +-
 src/couch_mrview_util.erl      |  34 +++++----
 test/08-changes_since.t        | 123 +++++++++++++++++++++++++++++++++
 4 files changed, 168 insertions(+), 22 deletions(-)
 create mode 100644 test/08-changes_since.t

diff --git a/src/couch_mrview_test_util.erl b/src/couch_mrview_test_util.erl
index 40ccf09..c68010c 100644
--- a/src/couch_mrview_test_util.erl
+++ b/src/couch_mrview_test_util.erl
@@ -40,14 +40,31 @@ save_docs(Db, Docs) ->
 
 
 make_docs(Count) ->
-    make_docs(Count, []).
-
-make_docs(Count, Acc) when Count =< 0 ->
-    Acc;
-make_docs(Count, Acc) ->
-    make_docs(Count-1, [doc(Count) | Acc]).
-
+    [doc(I) || I <- lists:seq(1, Count)].
 
+ddoc(changes) ->
+    couch_doc:from_json_obj({[
+        {<<"_id">>, <<"_design/bar">>},
+        {<<"options">>, {[
+            {<<"seq_indexed">>, true}
+        ]}},
+        {<<"views">>, {[
+            {<<"baz">>, {[
+                {<<"map">>, <<"function(doc) {emit(doc.val, doc.val);}">>}
+            ]}},
+            {<<"bing">>, {[
+                {<<"map">>, <<"function(doc) {}">>}
+            ]}},
+            {<<"zing">>, {[
+                {<<"map">>, <<
+                    "function(doc) {\n"
+                    "  if(doc.foo !== undefined)\n"
+                    "    emit(doc.foo, 0);\n"
+                    "}"
+                >>}
+            ]}}
+        ]}}
+    ]});
 ddoc(map) ->
     couch_doc:from_json_obj({[
         {<<"_id">>, <<"_design/bar">>},
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index f6557cf..16645aa 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -312,7 +312,7 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
                 SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
                 SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
                 RemSKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
-                RemKSs = [{[Seq, Key], DocId} || {Key, Seq, DocId} <- SToRem],
+                RemKSs = [{[Key, Seq], DocId} || {Key, Seq, DocId} <- SToRem],
                 SKVs1 = SKVs ++ SToAdd,
                 {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
                                                    SKVs1, RemSKs),
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 6d48ed0..f5fd66c 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -236,13 +236,8 @@ init_state(Db, Fd, State, Header) ->
         views=Views2
     }.
 
-less_json_seqs({SeqA, JsonA}, {SeqB, JsonB}) ->
-    case couch_ejson_compare:less(SeqA, SeqB) of
-        0 ->
-            couch_ejson_compare:less_json(JsonA, JsonB);
-        Result ->
-            Result < 0
-    end.
+less_json_seqs({SeqA, _JsonA}, {SeqB, _JsonB}) ->
+    couch_ejson_compare:less(SeqA, SeqB) < 0.
 
 open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
     FunSrcs = [FunSrc || {_Name, FunSrc} <- View#mrview.reduce_funs],
@@ -355,12 +350,23 @@ fold_fun(Fun, [KV|Rest], {KVReds, Reds}, Acc) ->
             {stop, Acc2}
     end.
 
+
 fold_changes(Bt, Fun, Acc, Opts) ->
     WrapperFun = fun(KV, _Reds, Acc2) ->
-        Fun(changes_expand_dups([KV], []), Acc2)
+        fold_changes_fun(Fun, changes_expand_dups([KV], []), Acc2)
     end,
     {ok, _LastRed, _Acc} = couch_btree:fold(Bt, WrapperFun, Acc, Opts).
 
+fold_changes_fun(_Fun, [], Acc) ->
+    {ok, Acc};
+fold_changes_fun(Fun, [KV|Rest],  Acc) ->
+    case Fun(KV, Acc) of
+        {ok, Acc2} ->
+            fold_changes_fun(Fun, Rest, Acc2);
+        {stop, Acc2} ->
+            {stop, Acc2}
+    end.
+
 
 fold_reduce({NthRed, Lang, View}, Fun,  Acc, Options) ->
     #mrview{
@@ -788,15 +794,15 @@ expand_dups([KV | Rest], Acc) ->
 changes_expand_dups([], Acc) ->
     lists:reverse(Acc);
 changes_expand_dups([{{[Key, Seq], DocId}, {dups, Vals}} | Rest], Acc) ->
-    Expanded = [{{Key, Seq, DocId}, Val} || Val <- Vals],
+    Expanded = [{{Seq, Key, DocId}, Val} || Val <- Vals],
     changes_expand_dups(Rest, Expanded ++ Acc);
-changes_expand_dups([{{Key, Seq}, {DocId, {dups, Vals}}} | Rest], Acc) ->
-    Expanded = [{{Key, Seq, DocId}, Val} || Val <- Vals],
+changes_expand_dups([{{Seq, Key}, {DocId, {dups, Vals}}} | Rest], Acc) ->
+    Expanded = [{{Seq, Key, DocId}, Val} || Val <- Vals],
     changes_expand_dups(Rest, Expanded ++ Acc);
 changes_expand_dups([{{[Key, Seq], DocId}, Val} | Rest], Acc) ->
-    changes_expand_dups(Rest, [{{Key, Seq, DocId}, Val} | Acc]);
-changes_expand_dups([{{Key, Seq}, {DocId, Val}} | Rest], Acc) ->
-    changes_expand_dups(Rest, [{{Key, Seq, DocId}, Val} | Acc]).
+    changes_expand_dups(Rest, [{{Seq, Key, DocId}, Val} | Acc]);
+changes_expand_dups([{{Seq, Key}, {DocId, Val}} | Rest], Acc) ->
+    changes_expand_dups(Rest, [{{Seq, Key, DocId}, Val} | Acc]).
 
 maybe_load_doc(_Db, _DI, #mrargs{include_docs=false}) ->
     [];
diff --git a/test/08-changes_since.t b/test/08-changes_since.t
new file mode 100644
index 0000000..3127cff
--- /dev/null
+++ b/test/08-changes_since.t
@@ -0,0 +1,123 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    etap:plan(10),
+    case (catch test()) of
+        ok ->
+            etap:end_tests();
+        Other ->
+            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+            etap:bail(Other)
+    end,
+    timer:sleep(300),
+    ok.
+
+test() ->
+    test_util:start_couch(),
+
+    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
+
+    test_basic(Db),
+    test_range(Db),
+    test_basic_since(Db),
+    test_range_since(Db),
+    test_basic_count(Db),
+    test_range_count(Db),
+    test_basic_count_since(Db),
+    test_range_count_since(Db),
+    test_compact(Db),
+    test_util:stop_couch(),
+    ok.
+
+
+test_basic(Db) ->
+    Result = run_query(Db, 0, []),
+    Expect = {ok, [
+                {{2, 1, <<"1">>}, 1},
+                {{3, 10, <<"10">>}, 10},
+                {{4, 2, <<"2">>}, 2},
+                {{5, 3, <<"3">>}, 3},
+                {{6, 4, <<"4">>}, 4},
+                {{7, 5, <<"5">>}, 5},
+                {{8, 6, <<"6">>}, 6},
+                {{9, 7, <<"7">>}, 7},
+                {{10, 8, <<"8">>}, 8},
+                {{11, 9, <<"9">>}, 9}
+    ]},
+    etap:is(Result, Expect, "Simple view query worked.").
+
+
+test_range(Db) ->
+    Result = run_query(Db, 0, [{start_key, 3}, {end_key, 5}]),
+    Expect = {ok, [
+                {{5, 3, <<"3">>}, 3},
+                {{6, 4, <<"4">>}, 4},
+                {{7, 5, <<"5">>}, 5}
+    ]},
+    etap:is(Result, Expect, "Query with range works.").
+
+test_basic_since(Db) ->
+    Result = run_query(Db, 5, []),
+    Expect = {ok, [
+                {{6, 4, <<"4">>}, 4},
+                {{7, 5, <<"5">>}, 5},
+                {{8, 6, <<"6">>}, 6},
+                {{9, 7, <<"7">>}, 7},
+                {{10, 8, <<"8">>}, 8},
+                {{11, 9, <<"9">>}, 9}
+    ]},
+    etap:is(Result, Expect, "Simple view query since 5 worked.").
+
+test_range_since(Db) ->
+    Result = run_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
+    Expect = {ok, [
+                {{6, 4, <<"4">>}, 4},
+                {{7, 5, <<"5">>}, 5}
+    ]},
+    etap:is(Result, Expect, "Query with range since 5 works.").
+
+test_basic_count(Db) ->
+    Result = run_count_query(Db, 0, []),
+    etap:is(Result, 10, "Simple view count worked.").
+
+test_range_count(Db) ->
+    Result = run_count_query(Db, 0, [{start_key, 3}, {end_key, 5}]),
+    etap:is(Result, 3, "Count with range works.").
+
+test_basic_count_since(Db) ->
+    Result = run_count_query(Db, 5, []),
+    etap:is(Result, 6, "Simple view count since 5 worked.").
+
+test_range_count_since(Db) ->
+    Result = run_count_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
+    etap:is(Result, 2, "Count with range since 5 works.").
+
+test_compact(Db) ->
+    Result = couch_mrview:compact(Db, <<"_design/bar">>),
+    etap:is(Result, ok, "compact view is OK"),
+    Count = run_count_query(Db, 0, []),
+    etap:is(Count, 10, "compact view worked.").
+
+run_query(Db, Since, Opts) ->
+    Fun = fun(KV, Acc) -> {ok, [KV | Acc]} end,
+    {ok, R} = couch_mrview:view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
+                                              Since, Fun, Opts, []),
+    {ok, lists:reverse(R)}.
+
+run_count_query(Db, Since, Opts) ->
+    couch_mrview:count_view_changes_since(Db, <<"_design/bar">>, <<"baz">>,
+                                          Since, Opts).

From 96562222256f1be7ec952f913aba8888628305eb Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Mon, 27 Jan 2014 21:03:28 +0100
Subject: [PATCH 06/41] couch_mrview: check removed keys from the index.

---
 test/08-changes_since.t | 42 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/test/08-changes_since.t b/test/08-changes_since.t
index 3127cff..5a10f4c 100644
--- a/test/08-changes_since.t
+++ b/test/08-changes_since.t
@@ -15,7 +15,7 @@
 % the License.
 
 main(_) ->
-    etap:plan(10),
+    etap:plan(14),
     case (catch test()) of
         ok ->
             etap:end_tests();
@@ -40,7 +40,8 @@ test() ->
     test_basic_count_since(Db),
     test_range_count_since(Db),
     test_compact(Db),
-    test_util:stop_couch(),
+    test_remove_key(Db),
+    catch test_util:stop_couch(),
     ok.
 
 
@@ -112,6 +113,43 @@ test_compact(Db) ->
     Count = run_count_query(Db, 0, []),
     etap:is(Count, 10, "compact view worked.").
 
+test_remove_key(Db) ->
+    %% add new doc
+    Doc = couch_mrview_test_util:doc(11),
+    {ok, Rev} = couch_db:update_doc(Db, Doc, []),
+    RevStr = couch_doc:rev_to_str(Rev),
+    {ok, _} =  couch_db:ensure_full_commit(Db),
+    {ok, Db1} = couch_db:reopen(Db),
+    Result = run_count_query(Db1, 0, []),
+    etap:is(Result, 11, "Add new doc worked."),
+    %% check new view key
+    Result1 = run_query(Db1, 0, [{start_key, 11}, {end_key, 11}]),
+    Expect = {ok, [
+                {{12, 11, <<"11">>}, 11}
+    ]},
+    etap:is(Result1, Expect, "added key OK."),
+
+    %% delete doc
+    Doc2 = couch_doc:from_json_obj({[
+                {<<"_id">>, <<"11">>},
+                {<<"_rev">>, RevStr},
+                {<<"_deleted">>, true}
+    ]}),
+    {ok, _} = couch_db:update_doc(Db1, Doc2, []),
+    {ok, Db2} = couch_db:reopen(Db1),
+    Result2 = run_count_query(Db2, 0, []),
+    etap:is(Result2, 11, "removed key saved."),
+    %% check new view key
+    Result3 = run_query(Db2, 0, [{start_key, 11}, {end_key, 11}]),
+    Expect2 = {ok, [
+                {{13, 11, <<"11">>}, {[{<<"_removed">>, true}]}}
+    ]},
+    etap:is(Result3, Expect2, "removed key OK.").
+
+
+
+
+
 run_query(Db, Since, Opts) ->
     Fun = fun(KV, Acc) -> {ok, [KV | Acc]} end,
     {ok, R} = couch_mrview:view_changes_since(Db, <<"_design/bar">>, <<"baz">>,

From 7a65916ba27c77d54191d9ac35009ad387aa0bf6 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Tue, 28 Jan 2014 23:23:37 +0100
Subject: [PATCH 07/41] couch_mrview: add couch_mrview:refresh/2 function

function to refresh a view index.

Conflicts:
	src/couch_mrview.erl
---
 src/couch_mrview.erl    | 19 +++++++++++++++++++
 test/08-changes_since.t |  4 ----
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 2f68fc0..8353961 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -19,6 +19,7 @@
 -export([count_view_changes_since/4, count_view_changes_since/5]).
 -export([get_info/2]).
 -export([trigger_update/2, trigger_update/3]).
+-export([refresh/2]).
 -export([compact/2, compact/3, cancel_compaction/2]).
 -export([cleanup/1]).
 
@@ -202,6 +203,24 @@ trigger_update(Db, DDoc, UpdateSeq) ->
     {ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
     couch_index:trigger_update(Pid, UpdateSeq).
 
+%% @doc refresh a view index
+refresh(#db{name=DbName}, DDoc) ->
+    refresh(DbName, DDoc);
+
+refresh(Db, DDoc) ->
+    UpdateSeq = couch_util:with_db(Db, fun(WDb) ->
+                    couch_db:get_update_seq(WDb)
+            end),
+
+    case couch_index_server:get_index(couch_mrview_index, Db, DDoc) of
+        {ok, Pid} ->
+            case catch couch_index:get_state(Pid, UpdateSeq) of
+                {ok, _} -> ok;
+                Error -> {error, Error}
+            end;
+        Error ->
+            {error, Error}
+    end.
 
 compact(Db, DDoc) ->
     compact(Db, DDoc, []).
diff --git a/test/08-changes_since.t b/test/08-changes_since.t
index 5a10f4c..58c92e9 100644
--- a/test/08-changes_since.t
+++ b/test/08-changes_since.t
@@ -146,10 +146,6 @@ test_remove_key(Db) ->
     ]},
     etap:is(Result3, Expect2, "removed key OK.").
 
-
-
-
-
 run_query(Db, Since, Opts) ->
     Fun = fun(KV, Acc) -> {ok, [KV | Acc]} end,
     {ok, R} = couch_mrview:view_changes_since(Db, <<"_design/bar">>, <<"baz">>,

From 18b5f6ff54683fca633cd1678522936a03598546 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Thu, 30 Jan 2014 09:21:06 +0100
Subject: [PATCH 08/41] add index update events notifications

Conflicts:
	src/couch_mrview_updater.erl
---
 src/couch_mrview_updater.erl |  3 +++
 test/09-index-events.t       | 46 ++++++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 test/09-index-events.t

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 16645aa..7b24e42 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -194,6 +194,9 @@ write_results(Parent, State) ->
                 Parent ! {new_state, NewState};
             true ->
                 send_partial(NewState#mrst.partial_resp_pid, NewState),
+                % notifify the view update
+                couch_index_event:notify({index_update, {DbName, IdxName,
+                                                         couch_mrview_index}}),
                 write_results(Parent, NewState)
             end
     end.
diff --git a/test/09-index-events.t b/test/09-index-events.t
new file mode 100644
index 0000000..90654b8
--- /dev/null
+++ b/test/09-index-events.t
@@ -0,0 +1,46 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    etap:plan(2),
+    case (catch test()) of
+        ok ->
+            etap:end_tests();
+        Other ->
+            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+            etap:bail(Other)
+    end,
+    timer:sleep(300),
+    ok.
+
+test() ->
+    test_util:start_couch(),
+    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
+    test_update_event(Db),
+    test_util:stop_couch(),
+    ok.
+
+test_update_event(Db) ->
+    {ok, Pid} = couch_index_event:start_link(self()),
+    etap:ok(is_pid(Pid), "event handler added"),
+    ok = couch_mrview:refresh(Db, <<"_design/bar">>),
+    Expect = {index_update, {<<"foo">>, <<"_design/bar">>,
+                             couch_mrview_index}},
+    receive
+        Event ->
+            etap:is(Event, Expect, "index update events OK")
+    end,
+    couch_index_event:stop(Pid).

From 1c24c425f2ec9fa63b0e01a13673af234043ee30 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Fri, 31 Jan 2014 13:13:23 +0100
Subject: [PATCH 09/41] couch_mrview: couch_mrview_changes:handle_changes

Similar to couch_changes:handle_changes but for view changes. It add
support for longpolling, normal and continuous stream The API differs
from the one for doc by beeing independant from the transport: the
support of HTTP will be added on top for example.

This API will be also used to replace the view filter in the current _changes
API.

Also add unittests.
---
 src/couch_mrview_changes.erl   | 173 +++++++++++++++++++++++++++++
 src/couch_mrview_test_util.erl |   2 +
 test/09-index-events.t         |  17 ++-
 test/10-index-changes.t        | 194 +++++++++++++++++++++++++++++++++
 4 files changed, 385 insertions(+), 1 deletion(-)
 create mode 100644 src/couch_mrview_changes.erl
 create mode 100644 test/10-index-changes.t

diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
new file mode 100644
index 0000000..2b8f910
--- /dev/null
+++ b/src/couch_mrview_changes.erl
@@ -0,0 +1,173 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+%
+-module(couch_mrview_changes).
+
+-export([handle_changes/6]).
+
+-include_lib("couch/include/couch_db.hrl").
+
+-record(vst, {dbname,
+              ddoc,
+              view,
+              view_options,
+              since,
+              callback,
+              acc,
+              user_timeout,
+              timeout,
+              heartbeat,
+              timeout_acc=0,
+              notifier,
+              stream}).
+
+-type changes_stream() :: true | false | once.
+-type changes_options() :: [{stream, changes_stream()} |
+                            {since, integer()} |
+                            {view_options, list()} |
+                            {timeout, integer()} |
+                            {heartbeat, true | integer()}].
+
+-export_type([changes_stream/0]).
+-export_type([changes_options/0]).
+
+%% @doc function returning changes in a streaming fashion if needed.
+-spec handle_changes(binary(), binary(), binary(), function(), term(),
+                     changes_options()) -> ok | {error, term()}.
+handle_changes(DbName, DDocId, View, Fun, Acc, Options) ->
+    Since = proplists:get_value(since, Options, 0),
+    Stream = proplists:get_value(stream, Options, false),
+    ViewOptions = proplists:get_value(view_options, Options, []),
+
+    State0 = #vst{dbname=DbName,
+                  ddoc=DDocId,
+                  view=View,
+                  view_options=ViewOptions,
+                  since=Since,
+                  callback=Fun,
+                  acc=Acc},
+
+    case view_changes_since(State0) of
+        {ok, #vst{since=LastSeq, acc=Acc2}=State} ->
+            case Stream of
+                true ->
+                    start_loop(State#vst{stream=true}, Options);
+                once when LastSeq =:= Since ->
+                    start_loop(State#vst{stream=once}, Options);
+                _ ->
+                    Fun(stop, {LastSeq, Acc2})
+            end;
+        {stop, #vst{since=LastSeq, acc=Acc2}} ->
+            Fun(stop, {LastSeq, Acc2});
+        Error ->
+            Error
+    end.
+
+start_loop(#vst{dbname=DbName, ddoc=DDocId}=State, Options) ->
+    {UserTimeout, Timeout, Heartbeat} = changes_timeout(Options),
+    Notifier = index_update_notifier(DbName, DDocId),
+    try
+        loop(State#vst{notifier=Notifier,
+                       user_timeout=UserTimeout,
+                       timeout=Timeout,
+                       heartbeat=Heartbeat})
+    after
+        couch_index_event:stop(Notifier)
+    end.
+
+loop(#vst{since=Since, callback=Callback, acc=Acc,
+          user_timeout=UserTimeout, timeout=Timeout,
+          heartbeat=Heartbeat, timeout_acc=TimeoutAcc,
+          stream=Stream}=State) ->
+    receive
+        index_update ->
+            case view_changes_since(State) of
+                {ok, State2} when Stream =:= true ->
+                    loop(State2#vst{timeout_acc=0});
+                {ok, #vst{since=LastSeq, acc=Acc2}} ->
+                    Callback(stop, {LastSeq, Acc2});
+                {stop, #vst{since=LastSeq, acc=Acc2}} ->
+                    Callback(stop, {LastSeq, Acc2})
+            end;
+        index_delete ->
+            Callback(stop, {Since, Acc})
+    after Timeout ->
+            TimeoutAcc2 = TimeoutAcc + Timeout,
+            case UserTimeout =< TimeoutAcc2 of
+                true ->
+                    Callback(stop, {Since, Acc});
+                false when Heartbeat =:= true ->
+                    case Callback(heartbeat, Acc) of
+                        {ok, Acc2} ->
+                            loop(State#vst{acc=Acc2, timeout_acc=TimeoutAcc2});
+                        {stop, Acc2} ->
+                            Callback(stop, {Since, Acc2})
+                    end;
+                _ ->
+                    Callback(stop, {Since, Acc})
+            end
+    end.
+
+changes_timeout(Options) ->
+    DefaultTimeout = list_to_integer(
+            couch_config:get("httpd", "changes_timeout", "60000")
+    ),
+    UserTimeout = proplists:get_value(timeout, Options, DefaultTimeout),
+    {Timeout, Heartbeat} = case proplists:get_value(heartbeat, Options) of
+        undefined -> {UserTimeout, false};
+        true ->
+            T = erlang:min(DefaultTimeout, UserTimeout),
+            {T, true};
+        H ->
+            T = erlang:min(H, UserTimeout),
+            {T, true}
+    end,
+    {UserTimeout, Timeout, Heartbeat}.
+
+view_changes_since(#vst{dbname=DbName, ddoc=DDocId, view=View,
+                        view_options=Options, since=Since,
+                        callback=Callback, acc=UserAcc}=State) ->
+    Wrapper = fun ({{Seq, _Key, _DocId}, _Val}=KV, {Go, Acc2, OldSeq}) ->
+            LastSeq = if OldSeq < Seq -> Seq;
+                true -> OldSeq
+            end,
+
+            case Callback(KV, Acc2) of
+                {ok, Acc3} -> {ok, {Go, Acc3, LastSeq}};
+                {stop, Acc3} -> {stop, {stop, Acc3, LastSeq}}
+            end
+    end,
+
+    Acc0 = {ok, UserAcc, Since},
+    case couch_mrview:view_changes_since(DbName, DDocId, View, Since,
+                                         Wrapper, Options, Acc0) of
+        {ok, {Go, UserAcc2, Since2}}->
+            {Go, State#vst{since=Since2, acc=UserAcc2}};
+        Error ->
+            Error
+    end.
+
+index_update_notifier(#db{name=DbName}, DDocId) ->
+    index_update_notifier(DbName, DDocId);
+index_update_notifier(DbName, DDocId) ->
+    Self = self(),
+    {ok, NotifierPid} = couch_index_event:start_link(fun
+                ({index_update, {Name, Id, couch_mrview_index}})
+                        when Name =:= DbName, Id =:= DDocId ->
+                    Self ! index_update;
+                ({index_delete, {Name, Id, couch_mrview_index}})
+                        when Name =:= DbName, Id =:= DDocId ->
+                    Self ! index_delete;
+                (_) ->
+                    ok
+            end),
+    NotifierPid.
diff --git a/src/couch_mrview_test_util.erl b/src/couch_mrview_test_util.erl
index c68010c..1d3d788 100644
--- a/src/couch_mrview_test_util.erl
+++ b/src/couch_mrview_test_util.erl
@@ -33,6 +33,8 @@ new_db(Name, Type) ->
     {ok, Db} = couch_db:create(Name, [?ADMIN_USER]),
     save_docs(Db, [ddoc(Type)]).
 
+delete_db(Name) ->
+    couch_server:delete(Name, [{user_ctx, ?ADMIN}]).
 
 save_docs(Db, Docs) ->
     {ok, _} = couch_db:update_docs(Db, Docs, []),
diff --git a/test/09-index-events.t b/test/09-index-events.t
index 90654b8..1489e4e 100644
--- a/test/09-index-events.t
+++ b/test/09-index-events.t
@@ -15,7 +15,7 @@
 % the License.
 
 main(_) ->
-    etap:plan(2),
+    etap:plan(4),
     case (catch test()) of
         ok ->
             etap:end_tests();
@@ -30,6 +30,7 @@ test() ->
     test_util:start_couch(),
     {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
     test_update_event(Db),
+    test_delete_event(Db),
     test_util:stop_couch(),
     ok.
 
@@ -44,3 +45,17 @@ test_update_event(Db) ->
             etap:is(Event, Expect, "index update events OK")
     end,
     couch_index_event:stop(Pid).
+
+test_delete_event(Db) ->
+     ok = couch_mrview:refresh(Db, <<"_design/bar">>),
+    {ok, Pid} = couch_index_event:start_link(self()),
+
+    etap:ok(is_pid(Pid), "event handler added"),
+    couch_mrview_test_util:delete_db(<<"foo">>),
+    Expect = {index_delete, {<<"foo">>, <<"_design/bar">>,
+                             couch_mrview_index}},
+    receive
+        Event ->
+            etap:is(Event, Expect, "index delete events OK")
+    end,
+    couch_index_event:stop(Pid).
diff --git a/test/10-index-changes.t b/test/10-index-changes.t
new file mode 100644
index 0000000..627376f
--- /dev/null
+++ b/test/10-index-changes.t
@@ -0,0 +1,194 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+main(_) ->
+    etap:plan(6),
+    case (catch test()) of
+        ok ->
+            etap:end_tests();
+        Other ->
+            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+            etap:bail(Other)
+    end,
+    timer:sleep(300),
+    ok.
+
+test() ->
+    test_util:start_couch(),
+    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
+    test_normal_changes(Db),
+    test_stream_once(Db),
+    test_stream_once_since(Db),
+    test_stream_once_timeout(Db),
+    test_stream_once_heartbeat(Db),
+    test_stream(Db),
+    test_util:stop_couch(),
+    ok.
+
+test_normal_changes(Db) ->
+    Result = run_query(Db, []),
+    Expect = {ok, 11, [
+                {{2, 1, <<"1">>}, 1},
+                {{3, 10, <<"10">>}, 10},
+                {{4, 2, <<"2">>}, 2},
+                {{5, 3, <<"3">>}, 3},
+                {{6, 4, <<"4">>}, 4},
+                {{7, 5, <<"5">>}, 5},
+                {{8, 6, <<"6">>}, 6},
+                {{9, 7, <<"7">>}, 7},
+                {{10, 8, <<"8">>}, 8},
+                {{11, 9, <<"9">>}, 9}
+    ]},
+    etap:is(Result, Expect, "normal changes worked.").
+
+test_stream_once(Db) ->
+    Result = run_query(Db, [{stream, once}]),
+    Expect = {ok, 11, [
+                {{2, 1, <<"1">>}, 1},
+                {{3, 10, <<"10">>}, 10},
+                {{4, 2, <<"2">>}, 2},
+                {{5, 3, <<"3">>}, 3},
+                {{6, 4, <<"4">>}, 4},
+                {{7, 5, <<"5">>}, 5},
+                {{8, 6, <<"6">>}, 6},
+                {{9, 7, <<"7">>}, 7},
+                {{10, 8, <<"8">>}, 8},
+                {{11, 9, <<"9">>}, 9}
+    ]},
+    etap:is(Result, Expect, "stream once since 0 worked.").
+
+
+test_stream_once_since(Db) ->
+    Self = self(),
+    spawn(fun() ->
+                Result = run_query(Db, [{since, 11},
+                                        {stream, once}]),
+                Self ! {result, Result}
+        end),
+
+    spawn(fun() ->
+                timer:sleep(1000),
+                {ok, Db1} = save_doc(Db, 11),
+                couch_mrview:refresh(Db1, <<"_design/bar">>)
+        end),
+
+    Expect = {ok,12,[{{12,11,<<"11">>},11}]},
+
+    receive
+        {result, Result} ->
+            etap:is(Result, Expect, "normal changes worked.")
+    after 5000 ->
+            io:format("never got the change", [])
+    end.
+
+
+test_stream_once_timeout(Db) ->
+    Self = self(),
+    spawn(fun() ->
+                Result = run_query(Db, [{since, 12},
+                                        {stream, once},
+                                        {timeout, 3000}]),
+                Self ! {result, Result}
+        end),
+
+
+
+    Expect = {ok, 12, []},
+
+    receive
+        {result, Result} ->
+            etap:is(Result, Expect, "got timeout.")
+    after 5000 ->
+            io:format("never got the change", [])
+    end.
+
+test_stream_once_heartbeat(Db) ->
+    Self = self(),
+    spawn(fun() ->
+                Result = run_query(Db, [{since, 12},
+                                        {stream, once},
+                                        {heartbeat, 1000}]),
+                Self ! {result, Result}
+        end),
+
+    spawn(fun() ->
+                timer:sleep(3000),
+                {ok, Db1} = save_doc(Db, 12),
+                couch_mrview:refresh(Db1, <<"_design/bar">>)
+        end),
+
+    Expect = {ok,13,[heartbeat,
+                     heartbeat,
+                     heartbeat,
+                     {{13,12,<<"12">>},12}]},
+
+
+
+    receive
+        {result, Result} ->
+            etap:is(Result, Expect, "heartbeat OK.")
+    after 5000 ->
+            io:format("never got the change", [])
+    end.
+
+
+test_stream(Db) ->
+    Self = self(),
+    spawn(fun() ->
+                Result = run_query(Db, [{since, 13},
+                                        stream,
+                                        {timeout, 3000}]),
+                Self ! {result, Result}
+        end),
+
+    spawn(fun() ->
+                timer:sleep(1000),
+                {ok, Db1} = save_doc(Db, 13),
+                couch_mrview:refresh(Db1, <<"_design/bar">>),
+                {ok, Db2} = save_doc(Db1, 14),
+                couch_mrview:refresh(Db2, <<"_design/bar">>)
+        end),
+
+    Expect = {ok, 15,[{{14,13,<<"13">>},13},
+                     {{15,14,<<"14">>},14}]},
+
+    receive
+        {result, Result} ->
+            etap:is(Result, Expect, "stream OK.")
+    after 5000 ->
+            io:format("never got the change", [])
+    end.
+
+
+save_doc(Db, Id) ->
+    Doc = couch_mrview_test_util:doc(Id),
+    {ok, _Rev} = couch_db:update_doc(Db, Doc, []),
+    {ok, _} =  couch_db:ensure_full_commit(Db),
+    couch_db:reopen(Db).
+
+run_query(Db, Opts) ->
+    Fun = fun
+        (stop, {LastSeq, Acc}) ->
+            {ok, LastSeq, Acc};
+        (heartbeat, Acc) ->
+            {ok, [heartbeat | Acc]};
+        (Event, Acc) ->
+            {ok, [Event | Acc]}
+    end,
+    couch_mrview:refresh(Db, <<"_design/bar">>),
+    {ok, LastSeq, R} = couch_mrview_changes:handle_changes(Db, <<"_design/bar">>,
+                                                  <<"baz">>, Fun, [], Opts),
+    {ok, LastSeq, lists:reverse(R)}.

From dfe991f0fcfe15740c0927c6be3cd9504114b894 Mon Sep 17 00:00:00 2001
From: benoitc <bchesneau@gmail.com>
Date: Fri, 7 Feb 2014 15:38:34 +0100
Subject: [PATCH 10/41] add supports of view changes in the _changes API

Now when the option `seq_indexed=true` is set in the design doc, the
view filter in _changes will use it to retrieve the results. Compared to
the current way, using a view index will be faster to retrieve changes.
It also gives the possibility to filter changes by key or get changes in
a key range. All the view options can be used.

Note 1: if someone is trying to filter a changes with view options when
the views are not indexed by sequence, a 400 error will be returned.
Note 2: The changes will only be returned when the view is updated if
seq_indexed=true

Conflicts:
	src/couch_mrview_http.erl
---
 src/couch_mrview_changes.erl |  6 ++----
 src/couch_mrview_http.erl    |  8 ++++++--
 src/couch_mrview_index.erl   | 20 +++++++++++++++++---
 test/09-index-events.t       | 21 +++++++++++++++++----
 4 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index 2b8f910..a0e5281 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -141,10 +141,8 @@ view_changes_since(#vst{dbname=DbName, ddoc=DDocId, view=View,
                 true -> OldSeq
             end,
 
-            case Callback(KV, Acc2) of
-                {ok, Acc3} -> {ok, {Go, Acc3, LastSeq}};
-                {stop, Acc3} -> {stop, {stop, Acc3, LastSeq}}
-            end
+            {Go, Acc3} = Callback(KV, Acc2),
+            {Go, {Go, Acc3, LastSeq}}
     end,
 
     Acc0 = {ok, UserAcc, Since},
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index aeaca8e..f3d3286 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -34,6 +34,10 @@
     check_view_etag/3
 ]).
 
+-export([parse_boolean/1,
+         parse_int/1,
+         parse_pos_int/1]).
+
 -include_lib("couch/include/couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 
@@ -438,8 +442,10 @@ parse_boolean(true) ->
     true;
 parse_boolean(false) ->
     false;
+
 parse_boolean(Val) when is_binary(Val) ->
     parse_boolean(?b2l(Val));
+
 parse_boolean(Val) ->
     case string:to_lower(Val) of
     "true" -> true;
@@ -449,7 +455,6 @@ parse_boolean(Val) ->
         throw({query_parse_error, ?l2b(Msg)})
     end.
 
-
 parse_int(Val) when is_integer(Val) ->
     Val;
 parse_int(Val) ->
@@ -461,7 +466,6 @@ parse_int(Val) ->
         throw({query_parse_error, ?l2b(Msg)})
     end.
 
-
 parse_pos_int(Val) ->
     case parse_int(Val) of
     IntVal when IntVal >= 0 ->
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 0b9b236..7598f13 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -39,8 +39,10 @@ get(Property, State) ->
             Opts = State#mrst.design_opts,
             IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
             LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
+            SeqIndexed = couch_util:get_value(<<"seq_indexed">>, Opts, false),
             if IncDesign -> [include_design]; true -> [] end
-                ++ if LocalSeq -> [local_seq]; true -> [] end;
+                ++ if LocalSeq -> [local_seq]; true -> [] end
+                ++ if SeqIndexed -> [seq_indexed]; true -> [] end;
         fd ->
             State#mrst.fd;
         language ->
@@ -56,13 +58,24 @@ get(Property, State) ->
                 language = Lang,
                 update_seq = UpdateSeq,
                 purge_seq = PurgeSeq,
-                views = Views
+                views = Views,
+                design_opts = Opts
             } = State,
             {ok, FileSize} = couch_file:bytes(Fd),
             {ok, ExternalSize} = couch_mrview_util:calculate_external_size(IdBtree,
                                                                            LogBtree,
                                                                            Views),
             ActiveSize = ExternalSize + couch_btree:size(Btree),
+
+            IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
+            LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
+            SeqIndexed = couch_util:get_value(<<"seq_indexed">>, Opts, false),
+            UpdateOptions =
+                if IncDesign -> [<<"include_design">>]; true -> [] end
+                ++ if LocalSeq -> [<<"local_seq">>]; true -> [] end
+                ++ if SeqIndexed -> [<<"seq_indexed">>]; true -> [] end,
+
+
             {ok, [
                 {signature, list_to_binary(couch_index_util:hexsig(Sig))},
                 {language, Lang},
@@ -74,7 +87,8 @@ get(Property, State) ->
                     {external, ExternalSize}
                 ]}},
                 {update_seq, UpdateSeq},
-                {purge_seq, PurgeSeq}
+                {purge_seq, PurgeSeq},
+                {update_options, UpdateOptions}
             ]};
         Other ->
             throw({unknown_index_property, Other})
diff --git a/test/09-index-events.t b/test/09-index-events.t
index 1489e4e..6cc1e9c 100644
--- a/test/09-index-events.t
+++ b/test/09-index-events.t
@@ -15,7 +15,7 @@
 % the License.
 
 main(_) ->
-    etap:plan(4),
+    etap:plan(5),
     case (catch test()) of
         ok ->
             etap:end_tests();
@@ -29,11 +29,18 @@ main(_) ->
 test() ->
     test_util:start_couch(),
     {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
+    test_info(Db),
     test_update_event(Db),
     test_delete_event(Db),
     test_util:stop_couch(),
     ok.
 
+test_info(Db) ->
+    {ok, Info} = couch_mrview:get_info(Db, <<"_design/bar">>),
+    etap:is(getval(update_options, Info), [<<"seq_indexed">>],
+            "update options OK"),
+    ok.
+
 test_update_event(Db) ->
     {ok, Pid} = couch_index_event:start_link(self()),
     etap:ok(is_pid(Pid), "event handler added"),
@@ -47,11 +54,13 @@ test_update_event(Db) ->
     couch_index_event:stop(Pid).
 
 test_delete_event(Db) ->
-     ok = couch_mrview:refresh(Db, <<"_design/bar">>),
+    ok = couch_mrview:refresh(Db, <<"_design/bar">>),
+    timer:sleep(300),
     {ok, Pid} = couch_index_event:start_link(self()),
+    etap:ok(is_pid(Pid), "delete event handler added"),
 
-    etap:ok(is_pid(Pid), "event handler added"),
-    couch_mrview_test_util:delete_db(<<"foo">>),
+
+    catch couch_mrview_test_util:delete_db(<<"foo">>),
     Expect = {index_delete, {<<"foo">>, <<"_design/bar">>,
                              couch_mrview_index}},
     receive
@@ -59,3 +68,7 @@ test_delete_event(Db) ->
             etap:is(Event, Expect, "index delete events OK")
     end,
     couch_index_event:stop(Pid).
+
+getval(Key, PL) ->
+    {value, {Key, Val}} = lists:keysearch(Key, 1, PL),
+    Val.

From 5691328fbd219d70760ddc5d579d0874c9ff2272 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Sat, 8 Feb 2014 19:55:40 +0100
Subject: [PATCH 11/41] couch_index: add background indexing facility

This change add the possibility to trigger a view indexation in
background. The indexation can only work in background if at least one
process acquired it using the `couch_index_server:acquire_index/3`
function. If all the process that acquired it are down or released it
using `couch_index_server:release_indexer/3` then the background task is
stopped.

By default the background indexation will happen every 1s or when 200
docs has been saved in the database. These parameters can be changed
using the options `threshold` and `refresh_interval` in the couch_index
section.

To use it with couch_mrview a new option {refresh, true} has been added
to couch_mrview_changes:handle_changes Also the query parameter
refresh=true is passsed in t the HTTP changes API.
---
 src/couch_mrview_changes.erl | 56 +++++++++++++++++++++++++-----------
 test/10-index-changes.t      | 17 ++++++++++-
 2 files changed, 56 insertions(+), 17 deletions(-)

diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index a0e5281..c31624e 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -28,14 +28,16 @@
               heartbeat,
               timeout_acc=0,
               notifier,
-              stream}).
+              stream,
+              refresh}).
 
 -type changes_stream() :: true | false | once.
 -type changes_options() :: [{stream, changes_stream()} |
                             {since, integer()} |
                             {view_options, list()} |
                             {timeout, integer()} |
-                            {heartbeat, true | integer()}].
+                            {heartbeat, true | integer()} |
+                            {refresh, true | false}].
 
 -export_type([changes_stream/0]).
 -export_type([changes_options/0]).
@@ -47,6 +49,7 @@ handle_changes(DbName, DDocId, View, Fun, Acc, Options) ->
     Since = proplists:get_value(since, Options, 0),
     Stream = proplists:get_value(stream, Options, false),
     ViewOptions = proplists:get_value(view_options, Options, []),
+    Refresh = proplists:get_value(refresh, Options, false),
 
     State0 = #vst{dbname=DbName,
                   ddoc=DDocId,
@@ -56,20 +59,25 @@ handle_changes(DbName, DDocId, View, Fun, Acc, Options) ->
                   callback=Fun,
                   acc=Acc},
 
-    case view_changes_since(State0) of
-        {ok, #vst{since=LastSeq, acc=Acc2}=State} ->
-            case Stream of
-                true ->
-                    start_loop(State#vst{stream=true}, Options);
-                once when LastSeq =:= Since ->
-                    start_loop(State#vst{stream=once}, Options);
-                _ ->
-                    Fun(stop, {LastSeq, Acc2})
-            end;
-        {stop, #vst{since=LastSeq, acc=Acc2}} ->
-            Fun(stop, {LastSeq, Acc2});
-        Error ->
-            Error
+    maybe_acquire_indexer(Refresh, DbName, DDocId),
+    try
+        case view_changes_since(State0) of
+            {ok, #vst{since=LastSeq, acc=Acc2}=State} ->
+                case Stream of
+                    true ->
+                        start_loop(State#vst{stream=true}, Options);
+                    once when LastSeq =:= Since ->
+                        start_loop(State#vst{stream=once}, Options);
+                    _ ->
+                        Fun(stop, {LastSeq, Acc2})
+                end;
+            {stop, #vst{since=LastSeq, acc=Acc2}} ->
+                Fun(stop, {LastSeq, Acc2});
+            Error ->
+                Error
+        end
+    after
+        maybe_release_indexer(Refresh, DbName, DDocId)
     end.
 
 start_loop(#vst{dbname=DbName, ddoc=DDocId}=State, Options) ->
@@ -169,3 +177,19 @@ index_update_notifier(DbName, DDocId) ->
                     ok
             end),
     NotifierPid.
+
+%% acquire the background indexing task so it can eventually be started
+%% if the process close the background task will be automatically
+%% released.
+maybe_acquire_indexer(false, _, _) ->
+    ok;
+maybe_acquire_indexer(true, DbName, DDocId) ->
+    couch_index_server:acquire_indexer(couch_mrview_index, DbName,
+                                       DDocId).
+
+%% release the background indexing task so it can eventually be stopped
+maybe_release_indexer(false, _, _) ->
+    ok;
+maybe_release_indexer(true, DbName, DDocId) ->
+    couch_index_server:release_indexer(couch_mrview_index, DbName,
+                                       DDocId).
diff --git a/test/10-index-changes.t b/test/10-index-changes.t
index 627376f..f53e9ed 100644
--- a/test/10-index-changes.t
+++ b/test/10-index-changes.t
@@ -15,7 +15,7 @@
 % the License.
 
 main(_) ->
-    etap:plan(6),
+    etap:plan(8),
     case (catch test()) of
         ok ->
             etap:end_tests();
@@ -35,6 +35,7 @@ test() ->
     test_stream_once_timeout(Db),
     test_stream_once_heartbeat(Db),
     test_stream(Db),
+    test_indexer(Db),
     test_util:stop_couch(),
     ok.
 
@@ -173,6 +174,20 @@ test_stream(Db) ->
     end.
 
 
+test_indexer(Db) ->
+    Result = run_query(Db, [{since, 14}]),
+    Expect = {ok, 15, [{{15,14,<<"14">>},14}]},
+    etap:is(Result, Expect, "refresh index by hand OK."),
+
+    {ok, Db1} = save_doc(Db, 15),
+    timer:sleep(1000),
+    Result1 = run_query(Db, [{since, 14}]),
+    Expect1 = {ok, 16, [{{15,14,<<"14">>},14},
+                       {{16,15,<<"15">>},15}]},
+    etap:is(Result1, Expect1, "changes indexed in background OK."),
+    ok.
+
+
 save_doc(Db, Id) ->
     Doc = couch_mrview_test_util:doc(Id),
     {ok, _Rev} = couch_db:update_doc(Db, Doc, []),

From 7bba23ffced699b8637116ad7f8e473b6b0b2d13 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Sat, 8 Feb 2014 20:20:47 +0100
Subject: [PATCH 12/41] couch_mrview: fix 10-index-changes.t

proper test to check the indexer.
---
 test/10-index-changes.t | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/test/10-index-changes.t b/test/10-index-changes.t
index f53e9ed..ce79eac 100644
--- a/test/10-index-changes.t
+++ b/test/10-index-changes.t
@@ -175,13 +175,13 @@ test_stream(Db) ->
 
 
 test_indexer(Db) ->
-    Result = run_query(Db, [{since, 14}]),
+    Result = run_query(Db, [{since, 14}, refresh]),
     Expect = {ok, 15, [{{15,14,<<"14">>},14}]},
     etap:is(Result, Expect, "refresh index by hand OK."),
 
     {ok, Db1} = save_doc(Db, 15),
-    timer:sleep(1000),
-    Result1 = run_query(Db, [{since, 14}]),
+    timer:sleep(1500),
+    Result1 = run_query(Db1, [{since, 14}], false),
     Expect1 = {ok, 16, [{{15,14,<<"14">>},14},
                        {{16,15,<<"15">>},15}]},
     etap:is(Result1, Expect1, "changes indexed in background OK."),
@@ -195,6 +195,9 @@ save_doc(Db, Id) ->
     couch_db:reopen(Db).
 
 run_query(Db, Opts) ->
+    run_query(Db, Opts, true).
+
+run_query(Db, Opts, Refresh) ->
     Fun = fun
         (stop, {LastSeq, Acc}) ->
             {ok, LastSeq, Acc};
@@ -203,7 +206,12 @@ run_query(Db, Opts) ->
         (Event, Acc) ->
             {ok, [Event | Acc]}
     end,
-    couch_mrview:refresh(Db, <<"_design/bar">>),
+    case Refresh of
+        true ->
+            couch_mrview:refresh(Db, <<"_design/bar">>);
+        false ->
+            ok
+    end,
     {ok, LastSeq, R} = couch_mrview_changes:handle_changes(Db, <<"_design/bar">>,
                                                   <<"baz">>, Fun, [], Opts),
     {ok, LastSeq, lists:reverse(R)}.

From a043479bc4182fa84190e7a2909bc1e344bd7236 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Sat, 8 Feb 2014 22:49:42 +0100
Subject: [PATCH 13/41] couch_mrview: add API to retrieve view info.

Add couch_mrview:get_view_info/3 to retrieve the internal informations
of a view like the last update seq in this view or the number of rows.

Conflicts:
	src/couch_mrview.erl
	test/04-index-info.t
---
 src/couch_mrview.erl | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 8353961..5b5ba25 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -19,6 +19,7 @@
 -export([count_view_changes_since/4, count_view_changes_since/5]).
 -export([get_info/2]).
 -export([trigger_update/2, trigger_update/3]).
+-export([get_view_info/3]).
 -export([refresh/2]).
 -export([compact/2, compact/3, cancel_compaction/2]).
 -export([cleanup/1]).
@@ -198,11 +199,33 @@ get_info(Db, DDoc) ->
 trigger_update(Db, DDoc) ->
     trigger_update(Db, DDoc, couch_db:get_update_seq(Db)).
 
-
 trigger_update(Db, DDoc, UpdateSeq) ->
     {ok, Pid} = couch_index_server:get_index(couch_mrview_index, Db, DDoc),
     couch_index:trigger_update(Pid, UpdateSeq).
 
+%% get informations on a view
+get_view_info(Db, DDoc, VName) ->
+    {ok, {_, View}, _, _Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+                                                          #mrargs{}),
+
+    %% get the total number of rows
+    {ok, TotalRows} =  couch_mrview_util:get_row_count(View),
+
+    %% get the total number of sequence logged in this view
+    SeqBtree = View#mrview.seq_btree,
+    {ok, TotalSeqs} = case SeqBtree of
+        nil -> {ok, 0};
+        _ ->
+            {ok, {Count, _Reds}} = couch_btree:full_reduce(SeqBtree),
+            {ok, Count}
+    end,
+
+    {ok, [{update_seq, View#mrview.update_seq},
+          {purge_seq, View#mrview.purge_seq},
+          {total_rows, TotalRows},
+          {total_seqs, TotalSeqs}]}.
+
+
 %% @doc refresh a view index
 refresh(#db{name=DbName}, DDoc) ->
     refresh(DbName, DDoc);

From 727ed6248428addfcf05c6c8951f636370084455 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Sat, 8 Feb 2014 23:15:28 +0100
Subject: [PATCH 14/41] couch_mrview: add HTTP handler to retrieve the infos on
 a view.

---
 src/couch_mrview.erl      |  6 +++---
 src/couch_mrview_http.erl | 12 ++++++++++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 5b5ba25..4658781 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -216,11 +216,11 @@ get_view_info(Db, DDoc, VName) ->
     {ok, TotalSeqs} = case SeqBtree of
         nil -> {ok, 0};
         _ ->
-            {ok, {Count, _Reds}} = couch_btree:full_reduce(SeqBtree),
-            {ok, Count}
+            couch_btree:full_reduce(SeqBtree)
     end,
 
-    {ok, [{update_seq, View#mrview.update_seq},
+    {ok, [{seq_indexed, View#mrview.seq_indexed},
+          {update_seq, View#mrview.update_seq},
           {purge_seq, View#mrview.purge_seq},
           {total_rows, TotalRows},
           {total_seqs, TotalSeqs}]}.
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index f3d3286..dfa294b 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -51,6 +51,18 @@ handle_all_docs_req(Req, _Db) ->
     couch_httpd:send_method_not_allowed(Req, "GET,POST,HEAD").
 
 
+handle_view_req(#httpd{method='GET',
+                      path_parts=[_, _, DDocName, _, VName, <<"_info">>]}=Req,
+                Db, DDoc) ->
+
+    DDocId = <<"_design/", DDocName/binary >>,
+    {ok, Info} = couch_mrview:get_view_info(Db#db.name, DDocId, VName),
+
+    FinalInfo = [{db_name, Db#db.name},
+                 {ddoc, DDocId},
+                 {view, VName}] ++ Info,
+    couch_httpd:send_json(Req, 200, {FinalInfo});
+
 handle_view_req(#httpd{method='GET'}=Req, Db, DDoc) ->
     [_, _, _, _, ViewName] = Req#httpd.path_parts,
     couch_stats:increment_counter([couchdb, httpd, view_reads]),

From d955e22a1fc15b0965cd3327abc69324f7602b17 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Sun, 9 Feb 2014 00:43:23 +0100
Subject: [PATCH 15/41] couch_replicator: add replication using changes in a
 view

Instead of a database, the replicator can now filter the documents using
a view index. All documents having a key emitted in the view can be
replicated.

View parameters can be used. Which means that you can replicate results
corresponding to a key in a view or a range.
---
 src/couch_mrview_http.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index dfa294b..fb2e4ac 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -53,7 +53,7 @@ handle_all_docs_req(Req, _Db) ->
 
 handle_view_req(#httpd{method='GET',
                       path_parts=[_, _, DDocName, _, VName, <<"_info">>]}=Req,
-                Db, DDoc) ->
+                Db, _DDoc) ->
 
     DDocId = <<"_design/", DDocName/binary >>,
     {ok, Info} = couch_mrview:get_view_info(Db#db.name, DDocId, VName),

From a1623552a84c3e66459a1c6afabec3f2ff89eca9 Mon Sep 17 00:00:00 2001
From: benoitc <bchesneau@gmail.com>
Date: Thu, 13 Feb 2014 22:59:47 +0100
Subject: [PATCH 16/41] use ERL_FLAGS instead of a shebang to launch tests

Conflicts:
	test/03-red-views.t
---
 test/08-changes_since.t | 1 -
 test/09-index-events.t  | 1 -
 test/10-index-changes.t | 1 -
 3 files changed, 3 deletions(-)

diff --git a/test/08-changes_since.t b/test/08-changes_since.t
index 58c92e9..3107c1a 100644
--- a/test/08-changes_since.t
+++ b/test/08-changes_since.t
@@ -1,6 +1,5 @@
 #!/usr/bin/env escript
 %% -*- erlang -*-
-%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
 
 % Licensed under the Apache License, Version 2.0 (the "License"); you may not
 % use this file except in compliance with the License. You may obtain a copy of
diff --git a/test/09-index-events.t b/test/09-index-events.t
index 6cc1e9c..26d904d 100644
--- a/test/09-index-events.t
+++ b/test/09-index-events.t
@@ -1,6 +1,5 @@
 #!/usr/bin/env escript
 %% -*- erlang -*-
-%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
 
 % Licensed under the Apache License, Version 2.0 (the "License"); you may not
 % use this file except in compliance with the License. You may obtain a copy of
diff --git a/test/10-index-changes.t b/test/10-index-changes.t
index ce79eac..0c5e924 100644
--- a/test/10-index-changes.t
+++ b/test/10-index-changes.t
@@ -1,6 +1,5 @@
 #!/usr/bin/env escript
 %% -*- erlang -*-
-%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
 
 % Licensed under the Apache License, Version 2.0 (the "License"); you may not
 % use this file except in compliance with the License. You may obtain a copy of

From 42ff200319232f3ce90ead2ff6d82c09cf02406e Mon Sep 17 00:00:00 2001
From: benoitc <bchesneau@gmail.com>
Date: Thu, 27 Feb 2014 21:55:46 +0100
Subject: [PATCH 17/41] fix case clause

---
 src/couch_mrview_changes.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index c31624e..735ded8 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -144,7 +144,7 @@ changes_timeout(Options) ->
 view_changes_since(#vst{dbname=DbName, ddoc=DDocId, view=View,
                         view_options=Options, since=Since,
                         callback=Callback, acc=UserAcc}=State) ->
-    Wrapper = fun ({{Seq, _Key, _DocId}, _Val}=KV, {Go, Acc2, OldSeq}) ->
+    Wrapper = fun ({{Seq, _Key, _DocId}, _Val}=KV, {_Go, Acc2, OldSeq}) ->
             LastSeq = if OldSeq < Seq -> Seq;
                 true -> OldSeq
             end,

From 142a4c3f99a28b83c753c29db4b4b27e8175ea4a Mon Sep 17 00:00:00 2001
From: benoitc <bchesneau@gmail.com>
Date: Tue, 4 Mar 2014 13:38:55 +0100
Subject: [PATCH 18/41] add /<db>/_design/<ddocname>/_reindex handler

When you 'POST' on the _reindex handle all the views of the group will
be reindexed.

close #RCOUCH-36
---
 src/couch_mrview_http.erl | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index fb2e4ac..7a4c13c 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -14,6 +14,7 @@
 
 -export([
     handle_all_docs_req/2,
+    handle_reindex_req/3,
     handle_view_req/3,
     handle_temp_view_req/2,
     handle_info_req/3,
@@ -50,6 +51,15 @@ handle_all_docs_req(#httpd{method='POST'}=Req, Db) ->
 handle_all_docs_req(Req, _Db) ->
     couch_httpd:send_method_not_allowed(Req, "GET,POST,HEAD").
 
+handle_reindex_req(#httpd{method='POST',
+                          path_parts=[_, _, DName,<<"_reindex">>]}=Req,
+                   Db, DDoc) ->
+    ok = couch_db:check_is_admin(Db),
+    couch_mrview:trigger_update(Db, <<"_design/", DName/binary>>),
+    couch_httpd:send_json(Req, 201, {[{<<"ok">>, true}]});
+handle_reindex_req(Req, _Db, _DDoc) ->
+    couch_httpd:send_method_not_allowed(Req, "POST").
+
 
 handle_view_req(#httpd{method='GET',
                       path_parts=[_, _, DDocName, _, VName, <<"_info">>]}=Req,
@@ -62,7 +72,6 @@ handle_view_req(#httpd{method='GET',
                  {ddoc, DDocId},
                  {view, VName}] ++ Info,
     couch_httpd:send_json(Req, 200, {FinalInfo});
-
 handle_view_req(#httpd{method='GET'}=Req, Db, DDoc) ->
     [_, _, _, _, ViewName] = Req#httpd.path_parts,
     couch_stats:increment_counter([couchdb, httpd, view_reads]),

From 45da4bc7c8b694aeaf3c1e0dbe4c8d04b92e9eaf Mon Sep 17 00:00:00 2001
From: benoitc <bchesneau@gmail.com>
Date: Sun, 2 Feb 2014 19:54:01 +0100
Subject: [PATCH 19/41] extract couch_httpd changes API in its own module

Conflicts:
	src/couch_mrview_updater.erl
---
 src/couch_mrview_updater.erl | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 7b24e42..16645aa 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -194,9 +194,6 @@ write_results(Parent, State) ->
                 Parent ! {new_state, NewState};
             true ->
                 send_partial(NewState#mrst.partial_resp_pid, NewState),
-                % notifify the view update
-                couch_index_event:notify({index_update, {DbName, IdxName,
-                                                         couch_mrview_index}}),
                 write_results(Parent, NewState)
             end
     end.

From 062c1e7fc9b0cb96f7a95b2b218d4a473db809e4 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Fri, 22 Aug 2014 15:52:31 +0700
Subject: [PATCH 20/41] Add preliminary version of view changes

---
 src/couch_mrview.erl         | 21 +++++++++++++++------
 src/couch_mrview_changes.erl | 12 +++++++++++-
 src/couch_mrview_http.erl    |  7 +++++++
 3 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index 4658781..c6e413b 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -15,6 +15,7 @@
 -export([validate/2]).
 -export([query_all_docs/2, query_all_docs/4]).
 -export([query_view/3, query_view/4, query_view/6]).
+-export([view_changes_since/5]).
 -export([view_changes_since/6, view_changes_since/7]).
 -export([count_view_changes_since/4, count_view_changes_since/5]).
 -export([get_info/2]).
@@ -139,13 +140,21 @@ query_view(Db, {Type, View, Ref}, Args, Callback, Acc) ->
         erlang:demonitor(Ref, [flush])
     end.
 
+view_changes_since(View, StartSeq, Fun, Opts0, Acc) ->
+    Wrapper = fun(KV, _, Acc1) ->
+        Fun(KV, Acc1)
+    end,
+    Opts = [{start_key, {StartSeq + 1, <<>>}}] ++ Opts0,
+    {ok, _LastRed, AccOut} = couch_btree:fold(View#mrview.seq_btree, Wrapper, Acc, Opts),
+    {ok, AccOut}.
+
 view_changes_since(Db, DDoc, VName, StartSeq, Fun, Acc) ->
     view_changes_since(Db, DDoc, VName, StartSeq, Fun, [], Acc).
 
 view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
     Args0 = make_view_changes_args(Options),
-    {ok, {_, View}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
-                                                          Args0),
+    {ok, {_, View, _}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+                                                             Args0),
     case View#mrview.seq_indexed of
         true ->
             OptList = make_view_changes_opts(StartSeq, Options, Args),
@@ -154,10 +163,10 @@ view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
                 _ -> View#mrview.seq_btree
             end,
             AccOut = lists:foldl(fun(Opts, Acc0) ->
-                        {ok, _R, A} = couch_mrview_util:fold_changes(
-                                    Btree, Fun, Acc0, Opts),
-                        A
-                end, Acc, OptList),
+                {ok, _R, A} = couch_mrview_util:fold_changes(
+                    Btree, Fun, Acc0, Opts),
+                A
+            end, Acc, OptList),
             {ok, AccOut};
         _ ->
             {error, seqs_not_indexed}
diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index 735ded8..cb55655 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -12,7 +12,7 @@
 %
 -module(couch_mrview_changes).
 
--export([handle_changes/6]).
+-export([handle_changes/6, handle_view_changes/5]).
 
 -include_lib("couch/include/couch_db.hrl").
 
@@ -42,6 +42,16 @@
 -export_type([changes_stream/0]).
 -export_type([changes_options/0]).
 
+handle_view_changes(Args, Req, Db, DDocId, ViewName) ->
+%    couch_index_server:acquire_indexer(couch_mrview_index, Db#db.name, DDocId),
+%    try
+    couch_changes:handle_changes(Args, Req, Db, {view, DDocId, ViewName}).
+%    after
+%        couch_index_server:release_indexer(couch_mrview_index, Db#db.name, DDocId)
+%    end.
+
+
+
 %% @doc function returning changes in a streaming fashion if needed.
 -spec handle_changes(binary(), binary(), binary(), function(), term(),
                      changes_options()) -> ok | {error, term()}.
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 7a4c13c..1e54195 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -14,6 +14,7 @@
 
 -export([
     handle_all_docs_req/2,
+    handle_view_changes_req/3,
     handle_reindex_req/3,
     handle_view_req/3,
     handle_temp_view_req/2,
@@ -61,6 +62,12 @@ handle_reindex_req(Req, _Db, _DDoc) ->
     couch_httpd:send_method_not_allowed(Req, "POST").
 
 
+handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_changes">>,ViewName]}=Req, Db, _DDoc) ->
+    ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db, true),
+    ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, DDocName, ViewName),
+    couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
+
+
 handle_view_req(#httpd{method='GET',
                       path_parts=[_, _, DDocName, _, VName, <<"_info">>]}=Req,
                 Db, _DDoc) ->

From c8394f90dd93210ea5a4a7acfb3b580cc0414bbe Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Fri, 22 Aug 2014 17:46:13 +0700
Subject: [PATCH 21/41] Make feed=continuous work for view changes

---
 src/couch_mrview_http.erl            |  2 +-
 src/couch_mrview_index.erl           |  4 ++-
 src/couch_mrview_update_notifier.erl | 49 ++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+), 2 deletions(-)
 create mode 100644 src/couch_mrview_update_notifier.erl

diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 1e54195..75c3cb5 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -64,7 +64,7 @@ handle_reindex_req(Req, _Db, _DDoc) ->
 
 handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_changes">>,ViewName]}=Req, Db, _DDoc) ->
     ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db, true),
-    ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, DDocName, ViewName),
+    ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, <<"_design/", DDocName/binary>>, ViewName),
     couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
 
 
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 7598f13..0e6ef6f 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -178,7 +178,9 @@ finish_update(State) ->
 
 commit(State) ->
     Header = {State#mrst.sig, couch_mrview_util:make_header(State)},
-    couch_file:write_header(State#mrst.fd, Header).
+    Resp = couch_file:write_header(State#mrst.fd, Header),
+    couch_mrview_update_notifier:notify({index_update, State#mrst.db_name, State#mrst.idx_name}),
+    Resp.
 
 
 compact(Db, State, Opts) ->
diff --git a/src/couch_mrview_update_notifier.erl b/src/couch_mrview_update_notifier.erl
new file mode 100644
index 0000000..1837b1f
--- /dev/null
+++ b/src/couch_mrview_update_notifier.erl
@@ -0,0 +1,49 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_mrview_update_notifier).
+
+-behaviour(gen_event).
+
+-export([start_link/1, notify/1]).
+-export([init/1, terminate/2, handle_event/2, handle_call/2, handle_info/2, code_change/3, stop/1]).
+
+-include_lib("couch/include/couch_db.hrl").
+
+start_link(Exec) ->
+    couch_event_sup:start_link(couch_mrview_update, {couch_mrview_update_notifier, make_ref()}, Exec).
+
+notify(Event) ->
+    gen_event:notify(couch_mrview_update, Event).
+
+stop(Pid) ->
+    couch_event_sup:stop(Pid).
+
+init(Fun) ->
+    {ok, Fun}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+handle_event(Event, Fun) ->
+    Fun(Event),
+    {ok, Fun}.
+
+handle_call(_Request, State) ->
+    {reply, ok, State}.
+
+handle_info({'EXIT', Pid, Reason}, Pid) ->
+    ?LOG_ERROR("View update notification process ~p died: ~p", [Pid, Reason]),
+    remove_handler.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.

From cc45433e9630b69d0e37fb3c5a893b7d1e1fa4e9 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Fri, 22 Aug 2014 20:28:44 +0700
Subject: [PATCH 22/41] Throw 400 on _view_changes when by-seq indexing is
 disabled

---
 src/couch_mrview_http.erl | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 75c3cb5..2b396b0 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -62,7 +62,20 @@ handle_reindex_req(Req, _Db, _DDoc) ->
     couch_httpd:send_method_not_allowed(Req, "POST").
 
 
-handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_changes">>,ViewName]}=Req, Db, _DDoc) ->
+handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_changes">>,ViewName]}=Req, Db, DDoc) ->
+    {DDocBody} = DDoc#doc.body,
+    case lists:keyfind(<<"options">>, 1, DDocBody) of
+        {<<"options">>, {Options}} when is_list(Options) ->
+            case lists:keyfind(<<"seq_indexed">>, 1, Options) of
+                {<<"seq_indexed">>, true} ->
+                    ok;
+                _ ->
+                    throw({bad_request, "view changes not enabled"})
+            end;
+        _ ->
+            throw({bad_request, "view changes not enabled"})
+    end,
+
     ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db, true),
     ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, <<"_design/", DDocName/binary>>, ViewName),
     couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).

From 24db15f39015bec4710da97789a1728523ce92e1 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Fri, 22 Aug 2014 23:57:48 +0700
Subject: [PATCH 23/41] Add view filtering optimization to changes feeds.
 Remove unnecessary old stuff.

---
 src/couch_mrview_changes.erl | 189 +----------------------------------
 src/couch_mrview_http.erl    |   6 +-
 2 files changed, 2 insertions(+), 193 deletions(-)

diff --git a/src/couch_mrview_changes.erl b/src/couch_mrview_changes.erl
index cb55655..ae5aa6e 100644
--- a/src/couch_mrview_changes.erl
+++ b/src/couch_mrview_changes.erl
@@ -12,194 +12,7 @@
 %
 -module(couch_mrview_changes).
 
--export([handle_changes/6, handle_view_changes/5]).
-
--include_lib("couch/include/couch_db.hrl").
-
--record(vst, {dbname,
-              ddoc,
-              view,
-              view_options,
-              since,
-              callback,
-              acc,
-              user_timeout,
-              timeout,
-              heartbeat,
-              timeout_acc=0,
-              notifier,
-              stream,
-              refresh}).
-
--type changes_stream() :: true | false | once.
--type changes_options() :: [{stream, changes_stream()} |
-                            {since, integer()} |
-                            {view_options, list()} |
-                            {timeout, integer()} |
-                            {heartbeat, true | integer()} |
-                            {refresh, true | false}].
-
--export_type([changes_stream/0]).
--export_type([changes_options/0]).
+-export([handle_view_changes/5]).
 
 handle_view_changes(Args, Req, Db, DDocId, ViewName) ->
-%    couch_index_server:acquire_indexer(couch_mrview_index, Db#db.name, DDocId),
-%    try
     couch_changes:handle_changes(Args, Req, Db, {view, DDocId, ViewName}).
-%    after
-%        couch_index_server:release_indexer(couch_mrview_index, Db#db.name, DDocId)
-%    end.
-
-
-
-%% @doc function returning changes in a streaming fashion if needed.
--spec handle_changes(binary(), binary(), binary(), function(), term(),
-                     changes_options()) -> ok | {error, term()}.
-handle_changes(DbName, DDocId, View, Fun, Acc, Options) ->
-    Since = proplists:get_value(since, Options, 0),
-    Stream = proplists:get_value(stream, Options, false),
-    ViewOptions = proplists:get_value(view_options, Options, []),
-    Refresh = proplists:get_value(refresh, Options, false),
-
-    State0 = #vst{dbname=DbName,
-                  ddoc=DDocId,
-                  view=View,
-                  view_options=ViewOptions,
-                  since=Since,
-                  callback=Fun,
-                  acc=Acc},
-
-    maybe_acquire_indexer(Refresh, DbName, DDocId),
-    try
-        case view_changes_since(State0) of
-            {ok, #vst{since=LastSeq, acc=Acc2}=State} ->
-                case Stream of
-                    true ->
-                        start_loop(State#vst{stream=true}, Options);
-                    once when LastSeq =:= Since ->
-                        start_loop(State#vst{stream=once}, Options);
-                    _ ->
-                        Fun(stop, {LastSeq, Acc2})
-                end;
-            {stop, #vst{since=LastSeq, acc=Acc2}} ->
-                Fun(stop, {LastSeq, Acc2});
-            Error ->
-                Error
-        end
-    after
-        maybe_release_indexer(Refresh, DbName, DDocId)
-    end.
-
-start_loop(#vst{dbname=DbName, ddoc=DDocId}=State, Options) ->
-    {UserTimeout, Timeout, Heartbeat} = changes_timeout(Options),
-    Notifier = index_update_notifier(DbName, DDocId),
-    try
-        loop(State#vst{notifier=Notifier,
-                       user_timeout=UserTimeout,
-                       timeout=Timeout,
-                       heartbeat=Heartbeat})
-    after
-        couch_index_event:stop(Notifier)
-    end.
-
-loop(#vst{since=Since, callback=Callback, acc=Acc,
-          user_timeout=UserTimeout, timeout=Timeout,
-          heartbeat=Heartbeat, timeout_acc=TimeoutAcc,
-          stream=Stream}=State) ->
-    receive
-        index_update ->
-            case view_changes_since(State) of
-                {ok, State2} when Stream =:= true ->
-                    loop(State2#vst{timeout_acc=0});
-                {ok, #vst{since=LastSeq, acc=Acc2}} ->
-                    Callback(stop, {LastSeq, Acc2});
-                {stop, #vst{since=LastSeq, acc=Acc2}} ->
-                    Callback(stop, {LastSeq, Acc2})
-            end;
-        index_delete ->
-            Callback(stop, {Since, Acc})
-    after Timeout ->
-            TimeoutAcc2 = TimeoutAcc + Timeout,
-            case UserTimeout =< TimeoutAcc2 of
-                true ->
-                    Callback(stop, {Since, Acc});
-                false when Heartbeat =:= true ->
-                    case Callback(heartbeat, Acc) of
-                        {ok, Acc2} ->
-                            loop(State#vst{acc=Acc2, timeout_acc=TimeoutAcc2});
-                        {stop, Acc2} ->
-                            Callback(stop, {Since, Acc2})
-                    end;
-                _ ->
-                    Callback(stop, {Since, Acc})
-            end
-    end.
-
-changes_timeout(Options) ->
-    DefaultTimeout = list_to_integer(
-            couch_config:get("httpd", "changes_timeout", "60000")
-    ),
-    UserTimeout = proplists:get_value(timeout, Options, DefaultTimeout),
-    {Timeout, Heartbeat} = case proplists:get_value(heartbeat, Options) of
-        undefined -> {UserTimeout, false};
-        true ->
-            T = erlang:min(DefaultTimeout, UserTimeout),
-            {T, true};
-        H ->
-            T = erlang:min(H, UserTimeout),
-            {T, true}
-    end,
-    {UserTimeout, Timeout, Heartbeat}.
-
-view_changes_since(#vst{dbname=DbName, ddoc=DDocId, view=View,
-                        view_options=Options, since=Since,
-                        callback=Callback, acc=UserAcc}=State) ->
-    Wrapper = fun ({{Seq, _Key, _DocId}, _Val}=KV, {_Go, Acc2, OldSeq}) ->
-            LastSeq = if OldSeq < Seq -> Seq;
-                true -> OldSeq
-            end,
-
-            {Go, Acc3} = Callback(KV, Acc2),
-            {Go, {Go, Acc3, LastSeq}}
-    end,
-
-    Acc0 = {ok, UserAcc, Since},
-    case couch_mrview:view_changes_since(DbName, DDocId, View, Since,
-                                         Wrapper, Options, Acc0) of
-        {ok, {Go, UserAcc2, Since2}}->
-            {Go, State#vst{since=Since2, acc=UserAcc2}};
-        Error ->
-            Error
-    end.
-
-index_update_notifier(#db{name=DbName}, DDocId) ->
-    index_update_notifier(DbName, DDocId);
-index_update_notifier(DbName, DDocId) ->
-    Self = self(),
-    {ok, NotifierPid} = couch_index_event:start_link(fun
-                ({index_update, {Name, Id, couch_mrview_index}})
-                        when Name =:= DbName, Id =:= DDocId ->
-                    Self ! index_update;
-                ({index_delete, {Name, Id, couch_mrview_index}})
-                        when Name =:= DbName, Id =:= DDocId ->
-                    Self ! index_delete;
-                (_) ->
-                    ok
-            end),
-    NotifierPid.
-
-%% acquire the background indexing task so it can eventually be started
-%% if the process close the background task will be automatically
-%% released.
-maybe_acquire_indexer(false, _, _) ->
-    ok;
-maybe_acquire_indexer(true, DbName, DDocId) ->
-    couch_index_server:acquire_indexer(couch_mrview_index, DbName,
-                                       DDocId).
-
-%% release the background indexing task so it can eventually be stopped
-maybe_release_indexer(false, _, _) ->
-    ok;
-maybe_release_indexer(true, DbName, DDocId) ->
-    couch_index_server:release_indexer(couch_mrview_index, DbName,
-                                       DDocId).
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 2b396b0..bb49583 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -36,10 +36,6 @@
     check_view_etag/3
 ]).
 
--export([parse_boolean/1,
-         parse_int/1,
-         parse_pos_int/1]).
-
 -include_lib("couch/include/couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 
@@ -76,7 +72,7 @@ handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_cha
             throw({bad_request, "view changes not enabled"})
     end,
 
-    ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db, true),
+    ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db),
     ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, <<"_design/", DDocName/binary>>, ViewName),
     couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
 

From b3fef4cbfcd02178248cedcef5057604ba2d7130 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Mon, 25 Aug 2014 16:49:07 +0700
Subject: [PATCH 24/41] Add rev to view changes respose

---
 src/couch_mrview_updater.erl | 40 +++++++++++++++++++-----------------
 src/couch_mrview_util.erl    |  2 +-
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 16645aa..3fe1dd6 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -124,11 +124,13 @@ process_doc(Doc, Seq, #mrst{doc_acc=Acc}=State) when length(Acc) > 100 ->
     couch_work_queue:queue(State#mrst.doc_queue, lists:reverse(Acc)),
     process_doc(Doc, Seq, State#mrst{doc_acc=[]});
 process_doc(nil, Seq, #mrst{doc_acc=Acc}=State) ->
-    {ok, State#mrst{doc_acc=[{nil, Seq, nil} | Acc]}};
-process_doc(#doc{id=Id, deleted=true}, Seq, #mrst{doc_acc=Acc}=State) ->
-    {ok, State#mrst{doc_acc=[{Id, Seq, deleted} | Acc]}};
+    {ok, State#mrst{doc_acc=[{nil, Seq, nil, nil} | Acc]}};
+process_doc(#doc{id=Id, deleted=true}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
+    {RevPos, [Rev | _]} = Doc#doc.revs,
+    {ok, State#mrst{doc_acc=[{Id, Seq, {RevPos, Rev}, deleted} | Acc]}};
 process_doc(#doc{id=Id}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
-    {ok, State#mrst{doc_acc=[{Id, Seq, Doc} | Acc]}}.
+    {RevPos, [Rev | _]} = Doc#doc.revs,
+    {ok, State#mrst{doc_acc=[{Id, Seq, {RevPos, Rev}, Doc} | Acc]}}.
 
 
 finish_update(#mrst{doc_acc=Acc}=State) ->
@@ -164,15 +166,15 @@ map_docs(Parent, State0) ->
             end,
             QServer = State1#mrst.qserver,
             DocFun = fun
-                ({nil, Seq, _}, {SeqAcc, Results}) ->
+                ({nil, Seq, _, _}, {SeqAcc, Results}) ->
                     {erlang:max(Seq, SeqAcc), Results};
-                ({Id, Seq, deleted}, {SeqAcc, Results}) ->
-                    {erlang:max(Seq, SeqAcc), [{Id, Seq, []} | Results]};
-                ({Id, Seq, Doc}, {SeqAcc, Results}) ->
+                ({Id, Seq, Rev, deleted}, {SeqAcc, Results}) ->
+                    {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, []} | Results]};
+                ({Id, Seq, Rev, Doc}, {SeqAcc, Results}) ->
                     couch_stats:increment_counter([couchdb, mrview, map_docs],
                                                   1),
                     {ok, Res} = couch_query_servers:map_doc_raw(QServer, Doc),
-                    {erlang:max(Seq, SeqAcc), [{Id, Seq, Res} | Results]}
+                    {erlang:max(Seq, SeqAcc), [{Id, Seq, Rev, Res} | Results]}
             end,
             FoldFun = fun(Docs, Acc) ->
                 update_task(length(Docs)),
@@ -250,27 +252,27 @@ merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Log) ->
                   Log1).
 
 
-merge_results({DocId, _Seq, []}, ViewKVs, DocIdKeys, Log) ->
-    {ViewKVs, [{DocId, []} | DocIdKeys], dict:store(DocId, [], Log)};
-merge_results({DocId, Seq, RawResults}, ViewKVs, DocIdKeys, Log) ->
+merge_results({DocId, _Seq, Rev, []}, ViewKVs, DocIdKeys, Log) ->
+    {ViewKVs, [{DocId, Rev, []} | DocIdKeys], dict:store(DocId, [], Log)};
+merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Log) ->
     JsonResults = couch_query_servers:raw_to_ejson(RawResults),
     Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
-    {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Results, ViewKVs, [],
+    {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [],
                                             [], Log),
     {ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}.
 
 
-insert_results(DocId, _Seq, [], [], ViewKVs, ViewIdKeys, Log) ->
+insert_results(DocId, _Seq, _Rev, [], [], ViewKVs, ViewIdKeys, Log) ->
     {lists:reverse(ViewKVs), {DocId, ViewIdKeys}, Log};
-insert_results(DocId, Seq, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVAcc,
+insert_results(DocId, Seq, Rev, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVAcc,
                VIdKeys, Log) ->
     CombineDupesFun = fun
         ({Key, Val}, {[{Key, {dups, Vals}} | Rest], IdKeys, Log2}) ->
             {[{Key, {dups, [Val | Vals]}} | Rest], IdKeys, Log2};
         ({Key, Val1}, {[{Key, Val2} | Rest], IdKeys, Log2}) ->
             {[{Key, {dups, [Val1, Val2]}} | Rest], IdKeys, Log2};
-        ({Key, _}=KV, {Rest, IdKeys, Log2}) ->
-            {[KV | Rest], [{Id, Key} | IdKeys],
+        ({Key, Value}, {Rest, IdKeys, Log2}) ->
+            {[{Key, Value} | Rest], [{Id, Key} | IdKeys],
              dict:append(DocId, {Id, {Key, Seq, add}}, Log2)}
     end,
     InitAcc = {[], VIdKeys, Log},
@@ -278,8 +280,8 @@ insert_results(DocId, Seq, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVAcc,
     {Duped, VIdKeys0, Log1} = lists:foldl(CombineDupesFun, InitAcc,
                                           lists:sort(KVs)),
     FinalKVs = [{{Key, DocId}, Val} || {Key, Val} <- Duped] ++ VKVs,
-    FinalSKVs = [{{Seq, Key}, {DocId, Val}} || {Key, Val} <- Duped] ++ SKVs,
-    insert_results(DocId, Seq, RKVs, RVKVs,
+    FinalSKVs = [{{Seq, Key}, {DocId, Val, Rev}} || {Key, Val} <- Duped] ++ SKVs,
+    insert_results(DocId, Seq, Rev, RKVs, RVKVs,
                   [{Id, {FinalKVs, FinalSKVs}} | VKVAcc], VIdKeys0, Log1).
 
 
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index f5fd66c..734ad26 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -854,7 +854,7 @@ mrverror(Mesg) ->
 
 
 to_key_seq(L) ->
-    [{{[Key, Seq], DocId}, Val} || {{Seq, Key}, {DocId, Val}} <- L].
+    [{{[Key, Seq], DocId}, {Val, Rev}} || {{Seq, Key}, {DocId, Val, Rev}} <- L].
 
 %% Updates 1.2.x or earlier view files to 1.3.x or later view files
 %% transparently, the first time the 1.2.x view file is opened by

From 690d878b26ce73ce713febed8a2ece4b89ba5306 Mon Sep 17 00:00:00 2001
From: benoitc <benoitc@apache.org>
Date: Mon, 27 Jan 2014 15:29:10 +0100
Subject: [PATCH 25/41] couch_mrview: fix compaction with seqs_indexed=true

This change smakes sure to compact the view indexes when using the view
changes.

Conflicts:
	src/couch_mrview_compactor.erl
---
 src/couch_mrview_compactor.erl | 84 +++++++++++++++++++++++++++++++---
 src/couch_mrview_util.erl      | 12 ++++-
 2 files changed, 88 insertions(+), 8 deletions(-)

diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index 8b1c8f7..b6a8e78 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -40,6 +40,8 @@ compact(State) ->
         sig=Sig,
         update_seq=Seq,
         id_btree=IdBtree,
+        log_btree=LogBtree,
+        seq_indexed=SeqIndexed,
         views=Views
     } = State,
     erlang:put(io_priority, {view_compact, DbName, IdxName}),
@@ -57,15 +59,28 @@ compact(State) ->
 
     #mrst{
         id_btree = EmptyIdBtree,
+        log_btree = EmptyLogBtree,
         views = EmptyViews
     } = EmptyState,
 
+    TotalChanges0 = case SeqIndexed of
+        true -> NumDocIds * 2;
+        _ -> NumDocIds
+    end,
+
     TotalChanges = lists:foldl(
         fun(View, Acc) ->
             {ok, Kvs} = couch_mrview_util:get_row_count(View),
-            Acc + Kvs
+            case SeqIndexed of
+                true ->
+                    {ok, SKvs} = couch_mrview_util:get_view_changes_count(View),
+                    Acc + Kvs + SKvs * 2;
+                false ->
+                    Acc + Kvs
+            end
         end,
-        NumDocIds, Views),
+        TotalChanges0, Views),
+
     couch_task_status:add_task([
         {type, view_compaction},
         {database, DbName},
@@ -107,13 +122,25 @@ compact(State) ->
     {ok, NewIdBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
     FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
 
+
+    {NewLogBtree, FinalAcc3} = case SeqIndexed of
+        true ->
+            compact_log(LogBtree, BufferSize,
+                        FinalAcc2#acc{kvs=[],
+                                      kvs_size=0,
+                                      btree=EmptyLogBtree});
+        _ ->
+            {nil, FinalAcc2}
+    end,
+
     {NewViews, _} = lists:mapfoldl(fun({View, EmptyView}, Acc) ->
         compact_view(View, EmptyView, BufferSize, Acc)
-    end, FinalAcc2, lists:zip(Views, EmptyViews)),
+    end, FinalAcc3, lists:zip(Views, EmptyViews)),
 
     unlink(EmptyState#mrst.fd),
     {ok, EmptyState#mrst{
         id_btree=NewIdBtree,
+        log_btree=NewLogBtree,
         views=NewViews,
         update_seq=Seq
     }}.
@@ -130,9 +157,53 @@ recompact(State) ->
             {ok, State2}
     end.
 
+compact_log(LogBtree, BufferSize, Acc0) ->
+    FoldFun = fun(KV, Acc) ->
+        #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc,
+        KvsSize2 = KvsSize + ?term_size(KV),
+        case KvsSize2 >= BufferSize of
+            true ->
+                {ok, Bt2} = couch_btree:add(Bt, lists:reverse([KV | Kvs])),
+                Acc2 = update_task(Acc, 1 + length(Kvs)),
+                {ok, Acc2#acc{
+                    btree = Bt2, kvs = [], kvs_size = 0}};
+            _ ->
+                {ok, Acc#acc{
+                    kvs = [KV | Kvs], kvs_size = KvsSize2}}
+        end
+    end,
+
+    {ok, _, FinalAcc} = couch_btree:foldl(LogBtree, FoldFun, Acc0),
+    #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
+    {ok, NewLogBtree} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
+    FinalAcc2 = update_task(FinalAcc, length(Uncopied)),
+    {NewLogBtree, FinalAcc2}.
 
 %% @spec compact_view(View, EmptyView, Retry, Acc) -> {CompactView, NewAcc}
 compact_view(#mrview{id_num=VID}=View, EmptyView, BufferSize, Acc0) ->
+
+    {NewBt, Acc1} = compact_view_btree(View#mrview.btree,
+                                       EmptyView#mrview.btree,
+                                       VID, BufferSize, Acc0),
+
+    %% are we indexing changes by sequences?
+    {NewSeqBt, NewKeyBySeqBt, FinalAcc} = case View#mrview.seq_indexed of
+        true ->
+            {SBt, Acc2} = compact_view_btree(View#mrview.seq_btree,
+                                             EmptyView#mrview.seq_btree,
+                                             VID, BufferSize, Acc1),
+            {KSBt, Acc3} = compact_view_btree(View#mrview.key_byseq_btree,
+                                              EmptyView#mrview.key_byseq_btree,
+                                              VID, BufferSize, Acc2),
+            {SBt, KSBt, Acc3};
+        _ ->
+            {nil, nil, Acc1}
+    end,
+    {EmptyView#mrview{btree=NewBt,
+                      seq_btree=NewSeqBt,
+                      key_byseq_btree=NewKeyBySeqBt}, FinalAcc}.
+
+compact_view_btree(Btree, EmptyBtree, VID, BufferSize, Acc0) ->
     Fun = fun(KV, #acc{btree = Bt, kvs = Kvs, kvs_size = KvsSize} = Acc) ->
         KvsSize2 = KvsSize + ?term_size(KV),
         if KvsSize2 >= BufferSize ->
@@ -144,13 +215,12 @@ compact_view(#mrview{id_num=VID}=View, EmptyView, BufferSize, Acc0) ->
         end
     end,
 
-    InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyView#mrview.btree},
-    {ok, _, FinalAcc} = couch_btree:foldl(View#mrview.btree, Fun, InitAcc),
+    InitAcc = Acc0#acc{kvs = [], kvs_size = 0, btree = EmptyBtree},
+    {ok, _, FinalAcc} = couch_btree:foldl(Btree, Fun, InitAcc),
     #acc{btree = Bt3, kvs = Uncopied} = FinalAcc,
     {ok, NewBt} = couch_btree:add(Bt3, lists:reverse(Uncopied)),
     FinalAcc2 = update_task(VID, FinalAcc, length(Uncopied)),
-    {EmptyView#mrview{btree=NewBt}, FinalAcc2}.
-
+    {NewBt, FinalAcc2}.
 
 update_task(Acc, ChangesInc) ->
     update_task(null, Acc, ChangesInc).
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 734ad26..7c639af 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -18,6 +18,7 @@
 -export([index_file/2, compaction_file/2, open_file/1]).
 -export([delete_files/2, delete_index_file/2, delete_compaction_file/2]).
 -export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]).
+-export([get_view_changes_count/1]).
 -export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
 -export([fold/4, fold_reduce/4]).
 -export([temp_view_to_ddoc/1]).
@@ -333,6 +334,12 @@ reduce_to_count(Reductions) ->
     {Count, _} = couch_btree:final_reduce(Reduce, Reductions),
     Count.
 
+%% @doc get all changes for a view
+get_view_changes_count(#mrview{seq_btree=Btree}) ->
+    couch_btree:fold_reduce(
+            Btree, fun(_SeqStart, PartialReds, 0) ->
+                    {ok, couch_btree:final_reduce(Btree, PartialReds)}
+            end,0, []).
 
 fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
     WrapperFun = fun(KV, Reds, Acc2) ->
@@ -634,10 +641,13 @@ reset_state(State) ->
     State#mrst{
         fd=nil,
         qserver=nil,
+        seq_indexed=State#mrst.seq_indexed,
         update_seq=0,
         id_btree=nil,
         log_btree=nil,
-        views=[View#mrview{btree=nil, seq_btree=nil, key_byseq_btree=nil}
+        views=[View#mrview{btree=nil, seq_btree=nil,
+                           key_byseq_btree=nil,
+                           seq_indexed=View#mrview.seq_indexed}
                || View <- State#mrst.views]
     }.
 

From c0eedd422415f36cf2d5bb68f7c72944fb373c79 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Tue, 26 Aug 2014 15:28:56 +0700
Subject: [PATCH 26/41] Make they key-by-key-and-seq view btree optional
 independently of the key-by-seq view btree

---
 include/couch_mrview.hrl       |   2 +
 src/couch_mrview.erl           |  31 ++++-----
 src/couch_mrview_compactor.erl |  30 +++++----
 src/couch_mrview_index.erl     |   4 ++
 src/couch_mrview_updater.erl   |  71 +++++++++++++--------
 src/couch_mrview_util.erl      | 111 ++++++++++++++++++++++-----------
 6 files changed, 159 insertions(+), 90 deletions(-)

diff --git a/include/couch_mrview.hrl b/include/couch_mrview.hrl
index f1911db..b8ea08a 100644
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@ -19,6 +19,7 @@
     language,
     design_opts=[],
     seq_indexed=false,
+    keyseq_indexed=false,
     lib,
     views,
     id_btree=nil,
@@ -45,6 +46,7 @@
     seq_btree=nil,
     key_byseq_btree=nil,
     seq_indexed=false,
+    keyseq_indexed=false,
     options=[]
 }).
 
diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index c6e413b..f6dec66 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -155,21 +155,22 @@ view_changes_since(Db, DDoc, VName, StartSeq, Fun, Options, Acc) ->
     Args0 = make_view_changes_args(Options),
     {ok, {_, View, _}, _, Args} = couch_mrview_util:get_view(Db, DDoc, VName,
                                                              Args0),
-    case View#mrview.seq_indexed of
-        true ->
-            OptList = make_view_changes_opts(StartSeq, Options, Args),
-            Btree = case is_key_byseq(Options) of
-                true -> View#mrview.key_byseq_btree;
-                _ -> View#mrview.seq_btree
-            end,
-            AccOut = lists:foldl(fun(Opts, Acc0) ->
-                {ok, _R, A} = couch_mrview_util:fold_changes(
-                    Btree, Fun, Acc0, Opts),
-                A
-            end, Acc, OptList),
-            {ok, AccOut};
-        _ ->
-            {error, seqs_not_indexed}
+    #mrview{seq_indexed=SIndexed, keyseq_indexed=KSIndexed} = View,
+    IsKSQuery = is_key_byseq(Options),
+    if (SIndexed andalso not IsKSQuery) orelse (KSIndexed andalso IsKSQuery) ->
+        OptList = make_view_changes_opts(StartSeq, Options, Args),
+        Btree = case IsKSQuery of
+            true -> View#mrview.key_byseq_btree;
+            _ -> View#mrview.seq_btree
+        end,
+        AccOut = lists:foldl(fun(Opts, Acc0) ->
+            {ok, _R, A} = couch_mrview_util:fold_changes(
+                Btree, Fun, Acc0, Opts),
+            A
+        end, Acc, OptList),
+        {ok, AccOut};
+    true ->
+        {error, seqs_not_indexed}
     end.
 
 count_view_changes_since(Db, DDoc, VName, SinceSeq) ->
diff --git a/src/couch_mrview_compactor.erl b/src/couch_mrview_compactor.erl
index b6a8e78..6d9382f 100644
--- a/src/couch_mrview_compactor.erl
+++ b/src/couch_mrview_compactor.erl
@@ -42,6 +42,7 @@ compact(State) ->
         id_btree=IdBtree,
         log_btree=LogBtree,
         seq_indexed=SeqIndexed,
+        keyseq_indexed=KeySeqIndexed,
         views=Views
     } = State,
     erlang:put(io_priority, {view_compact, DbName, IdxName}),
@@ -63,7 +64,7 @@ compact(State) ->
         views = EmptyViews
     } = EmptyState,
 
-    TotalChanges0 = case SeqIndexed of
+    TotalChanges0 = case SeqIndexed orelse KeySeqIndexed of
         true -> NumDocIds * 2;
         _ -> NumDocIds
     end,
@@ -71,10 +72,10 @@ compact(State) ->
     TotalChanges = lists:foldl(
         fun(View, Acc) ->
             {ok, Kvs} = couch_mrview_util:get_row_count(View),
-            case SeqIndexed of
+            case SeqIndexed orelse KeySeqIndexed of
                 true ->
                     {ok, SKvs} = couch_mrview_util:get_view_changes_count(View),
-                    Acc + Kvs + SKvs * 2;
+                    Acc + Kvs + SKvs;
                 false ->
                     Acc + Kvs
             end
@@ -187,18 +188,23 @@ compact_view(#mrview{id_num=VID}=View, EmptyView, BufferSize, Acc0) ->
                                        VID, BufferSize, Acc0),
 
     %% are we indexing changes by sequences?
-    {NewSeqBt, NewKeyBySeqBt, FinalAcc} = case View#mrview.seq_indexed of
+    {NewSeqBt, Acc2} = case View#mrview.seq_indexed of
         true ->
-            {SBt, Acc2} = compact_view_btree(View#mrview.seq_btree,
-                                             EmptyView#mrview.seq_btree,
-                                             VID, BufferSize, Acc1),
-            {KSBt, Acc3} = compact_view_btree(View#mrview.key_byseq_btree,
-                                              EmptyView#mrview.key_byseq_btree,
-                                              VID, BufferSize, Acc2),
-            {SBt, KSBt, Acc3};
+            compact_view_btree(View#mrview.seq_btree,
+                               EmptyView#mrview.seq_btree,
+                               VID, BufferSize, Acc1);
         _ ->
-            {nil, nil, Acc1}
+            {nil, Acc1}
     end,
+    {NewKeyBySeqBt, FinalAcc} = case View#mrview.keyseq_indexed of
+        true ->
+            compact_view_btree(View#mrview.key_byseq_btree,
+                               EmptyView#mrview.key_byseq_btree,
+                               VID, BufferSize, Acc2);
+        _ ->
+            {nil, Acc2}
+    end,
+
     {EmptyView#mrview{btree=NewBt,
                       seq_btree=NewSeqBt,
                       key_byseq_btree=NewKeyBySeqBt}, FinalAcc}.
diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 0e6ef6f..63340cb 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -40,8 +40,10 @@ get(Property, State) ->
             IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
             LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
             SeqIndexed = couch_util:get_value(<<"seq_indexed">>, Opts, false),
+            KeySeqIndexed = couch_util:get_value(<<"keyseq_indexed">>, Opts, false),
             if IncDesign -> [include_design]; true -> [] end
                 ++ if LocalSeq -> [local_seq]; true -> [] end
+                ++ if KeySeqIndexed -> [keyseq_indexed]; true -> [] end
                 ++ if SeqIndexed -> [seq_indexed]; true -> [] end;
         fd ->
             State#mrst.fd;
@@ -70,9 +72,11 @@ get(Property, State) ->
             IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
             LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
             SeqIndexed = couch_util:get_value(<<"seq_indexed">>, Opts, false),
+            KeySeqIndexed = couch_util:get_value(<<"keyseq_indexed">>, Opts, false),
             UpdateOptions =
                 if IncDesign -> [<<"include_design">>]; true -> [] end
                 ++ if LocalSeq -> [<<"local_seq">>]; true -> [] end
+                ++ if KeySeqIndexed -> [<<"keyseq_indexed">>]; true -> [] end
                 ++ if SeqIndexed -> [<<"seq_indexed">>]; true -> [] end,
 
 
diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 3fe1dd6..2a67dff 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -86,27 +86,37 @@ purge(_Db, PurgeSeq, PurgedIdRevs, State) ->
     SeqsToRemove = lists:foldl(MakeDictFun, dict:new(), LLookups),
 
     RemKeysFun = fun(#mrview{id_num=ViewId}=View) ->
+        #mrview{seq_indexed=SIndexed, keyseq_indexed=KSIndexed} = View,
         ToRem = couch_util:dict_find(ViewId, KeysToRemove, []),
         {ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, [], ToRem),
         NewPurgeSeq = case VBtree2 =/= View#mrview.btree of
             true -> PurgeSeq;
             _ -> View#mrview.purge_seq
         end,
-        {SeqBtree2, KeyBySeqBtree2} = case View#mrview.seq_indexed of
-            true ->
-                SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+        {SeqBtree3, KeyBySeqBtree3} = if SIndexed orelse KSIndexed ->
+            SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+            {ok, SeqBtree2} = if SIndexed ->
                 SKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+                couch_btree:add_remove(View#mrview.seq_btree,
+                                       [], SKs);
+            true ->
+                {ok, nil}
+            end,
+            {ok, KeyBySeqBtree2} = if KSIndexed ->
                 KSs = [{[Seq, Key], DocId} || {Key, Seq, DocId} <- SToRem],
-                {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
-                                                   [], SKs),
-                {ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
-                                                    [], KSs),
-                {SBt, KSbt};
-            _ -> {nil, nil}
+                couch_btree:add_remove(View#mrview.key_byseq_btree,
+                                       [], KSs);
+            true ->
+                {ok, nil}
+            end,
+            {SeqBtree2, KeyBySeqBtree2};
+        true ->
+            {nil, nil}
         end,
+
         View#mrview{btree=VBtree2,
-                    seq_btree=SeqBtree2,
-                    key_byseq_btree=KeyBySeqBtree2,
+                    seq_btree=SeqBtree3,
+                    key_byseq_btree=KeyBySeqBtree3,
                     purge_seq=NewPurgeSeq}
 
     end,
@@ -301,6 +311,7 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
     end,
 
     UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs, SKVs}}) ->
+        #mrview{seq_indexed=SIndexed, keyseq_indexed=KSIndexed} = View,
         ToRem = couch_util:dict_find(ViewId, ToRemByView, []),
         {ok, VBtree2} = couch_btree:add_remove(View#mrview.btree, KVs, ToRem),
         NewUpdateSeq = case VBtree2 =/= View#mrview.btree of
@@ -309,25 +320,33 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
         end,
 
         %% store the view changes.
-        {SeqBtree2, KeyBySeqBtree2} = case View#mrview.seq_indexed of
-            true ->
-                SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
-                SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
+        {SeqBtree3, KeyBySeqBtree3} = if SIndexed orelse KSIndexed ->
+            SToRem = couch_util:dict_find(ViewId, SeqsToRemove, []),
+            SToAdd = couch_util:dict_find(ViewId, SeqsToAdd, []),
+            SKVs1 = SKVs ++ SToAdd,
+
+            {ok, SeqBtree2} = if SIndexed ->
                 RemSKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+                couch_btree:add_remove(View#mrview.seq_btree,
+                                       SKVs1, RemSKs);
+            true ->
+                {ok, nil}
+            end,
+            {ok, KeyBySeqBtree2} = if KSIndexed ->
                 RemKSs = [{[Key, Seq], DocId} || {Key, Seq, DocId} <- SToRem],
-                SKVs1 = SKVs ++ SToAdd,
-                {ok, SBt} = couch_btree:add_remove(View#mrview.seq_btree,
-                                                   SKVs1, RemSKs),
-
-                {ok, KSbt} = couch_btree:add_remove(View#mrview.key_byseq_btree,
-                                                    couch_mrview_util:to_key_seq(SKVs1),
-                                                    RemKSs),
-                {SBt, KSbt};
-            _ -> {nil, nil}
+                couch_btree:add_remove(View#mrview.key_byseq_btree,
+                                       couch_mrview_util:to_key_seq(SKVs1),
+                                       RemKSs);
+            true ->
+                {ok, nil}
+            end,
+            {SeqBtree2, KeyBySeqBtree2};
+        true ->
+            {nil, nil}
         end,
         View#mrview{btree=VBtree2,
-                    seq_btree=SeqBtree2,
-                    key_byseq_btree=KeyBySeqBtree2,
+                    seq_btree=SeqBtree3,
+                    key_byseq_btree=KeyBySeqBtree3,
                     update_seq=NewUpdateSeq}
     end,
 
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 7c639af..ce037d5 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -97,12 +97,13 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
     end,
     {DesignOpts} = proplists:get_value(<<"options">>, Fields, {[]}),
     SeqIndexed = proplists:get_value(<<"seq_indexed">>, DesignOpts, false),
+    KeySeqIndexed = proplists:get_value(<<"keyseq_indexed">>, DesignOpts, false),
 
     {RawViews} = couch_util:get_value(<<"views">>, Fields, {[]}),
     BySrc = lists:foldl(MakeDict, dict:new(), RawViews),
 
     NumViews = fun({_, View}, N) ->
-            {View#mrview{id_num=N, seq_indexed=SeqIndexed}, N+1}
+            {View#mrview{id_num=N, seq_indexed=SeqIndexed, keyseq_indexed=KeySeqIndexed}, N+1}
     end,
     {Views, _} = lists:mapfoldl(NumViews, 0, lists:sort(dict:to_list(BySrc))),
 
@@ -116,7 +117,8 @@ ddoc_to_mrst(DbName, #doc{id=Id, body={Fields}}) ->
         views=Views,
         language=Language,
         design_opts=DesignOpts,
-        seq_indexed=SeqIndexed
+        seq_indexed=SeqIndexed,
+        keyseq_indexed=KeySeqIndexed
     },
     SigInfo = {Views, Language, DesignOpts, couch_index_util:sort_lib(Lib)},
     {ok, IdxState#mrst{sig=couch_util:md5(term_to_binary(SigInfo))}}.
@@ -161,7 +163,7 @@ view_sig(Db, State, View, #mrargs{include_docs=true}=Args) ->
     UpdateSeq = couch_db:get_update_seq(Db),
     PurgeSeq = couch_db:get_purge_seq(Db),
     Bin = term_to_binary({BaseSig, UpdateSeq, PurgeSeq,
-                          State#mrst.seq_indexed}),
+                          State#mrst.seq_indexed, State#mrst.keyseq_indexed}),
     couch_index_util:hexsig(couch_util:md5(Bin));
 view_sig(Db, State, {_Nth, _Lang, View}, Args) ->
     view_sig(Db, State, View, Args);
@@ -170,11 +172,12 @@ view_sig(_Db, State, View, Args0) ->
     UpdateSeq = View#mrview.update_seq,
     PurgeSeq = View#mrview.purge_seq,
     SeqIndexed = View#mrview.seq_indexed,
+    KeySeqIndexed = View#mrview.keyseq_indexed,
     Args = Args0#mrargs{
         preflight_fun=undefined,
         extra=[]
     },
-    Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, SeqIndexed, Args}),
+    Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args}),
     couch_index_util:hexsig(couch_util:md5(Bin)).
 
 
@@ -202,7 +205,12 @@ init_state(Db, Fd, State, #index_header{
         view_states=[{Bt, nil, nil, USeq, PSeq} || {Bt, USeq, PSeq} <- ViewStates]
         });
 init_state(Db, Fd, State, Header) ->
-    #mrst{language=Lang, views=Views, seq_indexed=SeqIndexed} = State,
+    #mrst{
+        language=Lang,
+        views=Views,
+        seq_indexed=SeqIndexed,
+        keyseq_indexed=KeySeqIndexed
+    } = State,
     #mrheader{
         seq=Seq,
         purge_seq=PurgeSeq,
@@ -219,7 +227,7 @@ init_state(Db, Fd, State, Header) ->
 
     IdBtOpts = [{compression, couch_db:compression(Db)}],
     {ok, IdBtree} = couch_btree:open(IdBtreeState, Fd, IdBtOpts),
-    {ok, LogBtree} = case SeqIndexed of
+    {ok, LogBtree} = case SeqIndexed orelse KeySeqIndexed of
         true -> couch_btree:open(LogBtreeState, Fd, IdBtOpts);
         false -> {ok, nil}
     end,
@@ -266,20 +274,23 @@ open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
     ],
     {ok, Btree} = couch_btree:open(BTState, Fd, ViewBtOpts),
 
-    {SeqBtree, KeyBySeqBtree} = case View#mrview.seq_indexed of
-        true ->
-            BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
-            ViewSeqBtOpts = [{less, fun less_json_seqs/2},
-                             {reduce, BySeqReduceFun},
-                             {compression, couch_db:compression(Db)}],
-            KeyBySeqBtOpts = [{less, Less},
-                              {reduce, BySeqReduceFun},
-                              {compression, couch_db:compression(Db)}],
-            {ok, SBt} = couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts),
-            {ok, KSBt} = couch_btree:open(KSeqBTState, Fd, KeyBySeqBtOpts),
-            {SBt, KSBt};
-        false ->
-            {nil, nil}
+    BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
+    {ok, SeqBtree} = if View#mrview.seq_indexed ->
+        ViewSeqBtOpts = [{less, fun less_json_seqs/2},
+                         {reduce, BySeqReduceFun},
+                         {compression, couch_db:compression(Db)}],
+
+        couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts);
+    true ->
+        {ok, nil}
+    end,
+    {ok, KeyBySeqBtree} = if View#mrview.keyseq_indexed ->
+        KeyBySeqBtOpts = [{less, Less},
+                          {reduce, BySeqReduceFun},
+                          {compression, couch_db:compression(Db)}],
+        couch_btree:open(KSeqBTState, Fd, KeyBySeqBtOpts);
+    true ->
+        {ok, nil}
     end,
 
     View#mrview{btree=Btree,
@@ -335,11 +346,25 @@ reduce_to_count(Reductions) ->
     Count.
 
 %% @doc get all changes for a view
-get_view_changes_count(#mrview{seq_btree=Btree}) ->
-    couch_btree:fold_reduce(
-            Btree, fun(_SeqStart, PartialReds, 0) ->
-                    {ok, couch_btree:final_reduce(Btree, PartialReds)}
-            end,0, []).
+get_view_changes_count(View) ->
+    #mrview{seq_btree=SBtree, key_byseq_btree=KSBtree} = View,
+    CountFun = fun(_SeqStart, PartialReds, 0) ->
+        {ok, couch_btree:final_reduce(SBtree, PartialReds)}
+    end,
+    {ok, Count} = case {SBtree, KSBtree} of
+        {nil, nil} ->
+            {ok, 0};
+        {#btree{}, nil} ->
+            couch_btree:fold_reduce(SBtree, CountFun, 0, []);
+        {nil, #btree{}} ->
+            couch_btree:fold_reduce(KSBtree, CountFun, 0, [])
+    end,
+    case {SBtree, KSBtree} of
+        {#btree{}, #btree{}} ->
+            {ok, Count*2};
+        _ ->
+            {ok, Count}
+    end.
 
 fold(#mrview{btree=Bt}, Fun, Acc, Opts) ->
     WrapperFun = fun(KV, Reds, Acc2) ->
@@ -563,11 +588,17 @@ make_header(State) ->
     } = State,
 
     ViewStates = lists:foldr(fun(V, Acc) ->
-                    {SeqBtState, KSeqBtState} = case V#mrview.seq_indexed of
+                    SeqBtState = case V#mrview.seq_indexed of
+                        true ->
+                            couch_btree:get_state(V#mrview.seq_btree);
+                        _ ->
+                            nil
+                    end,
+                    KSeqBtState = case V#mrview.keyseq_indexed of
                         true ->
-                            {couch_btree:get_state(V#mrview.seq_btree),
-                             couch_btree:get_state(V#mrview.key_byseq_btree)};
-                        _ -> {nil, nil}
+                            couch_btree:get_state(V#mrview.key_byseq_btree);
+                        _ ->
+                            nil
                     end,
                     [{couch_btree:get_state(V#mrview.btree),
                       SeqBtState,
@@ -642,12 +673,14 @@ reset_state(State) ->
         fd=nil,
         qserver=nil,
         seq_indexed=State#mrst.seq_indexed,
+        keyseq_indexed=State#mrst.keyseq_indexed,
         update_seq=0,
         id_btree=nil,
         log_btree=nil,
         views=[View#mrview{btree=nil, seq_btree=nil,
                            key_byseq_btree=nil,
-                           seq_indexed=View#mrview.seq_indexed}
+                           seq_indexed=View#mrview.seq_indexed,
+                           keyseq_indexed=View#mrview.keyseq_indexed}
                || View <- State#mrst.views]
     }.
 
@@ -757,14 +790,18 @@ changes_ekey_opts(_StartSeq, #mrargs{end_key=EKey,
 
 
 
+
 calculate_external_size(IdBt, LogBt, Views) ->
-    SumFun = fun
-        (#mrview{btree=Bt, seq_btree=nil}, Acc) ->
-            sum_btree_sizes(Acc, couch_btree:size(Bt));
-        (#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
-            Acc1 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
-            Acc2 = sum_btree_sizes(Acc1, couch_btree:size(SBt)),
-            sum_btree_sizes(Acc2, couch_btree:size(KSBt))
+    SumFun = fun(#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
+        Size0 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
+        Size1 = case SBt of
+            nil -> Size0;
+            _ -> sum_btree_sizes(Size0, couch_btree:size(SBt))
+        end,
+        case KSBt of
+            nil -> Size1;
+            _ -> sum_btree_sizes(Size1, couch_btree:size(KSBt))
+        end
     end,
     Size = case LogBt of
         nil ->

From c31b07856af60db0873d53c469798fd35ea0ca69 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Thu, 28 Aug 2014 00:02:48 +0700
Subject: [PATCH 27/41] Change function names to match changes in couchdb-couch

---
 src/couch_mrview_http.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index bb49583..fa933e7 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -72,9 +72,9 @@ handle_view_changes_req(#httpd{path_parts=[_,<<"_design">>,DDocName,<<"_view_cha
             throw({bad_request, "view changes not enabled"})
     end,
 
-    ChangesArgs = couch_httpd_changes:parse_changes_query(Req, Db),
+    ChangesArgs = couch_httpd_db:parse_changes_query(Req, Db),
     ChangesFun = couch_mrview_changes:handle_view_changes(ChangesArgs, Req, Db, <<"_design/", DDocName/binary>>, ViewName),
-    couch_httpd_changes:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
+    couch_httpd_db:handle_changes_req(Req, Db, ChangesArgs, ChangesFun).
 
 
 handle_view_req(#httpd{method='GET',

From c0ad453e7c467557a2237e109d2c5af0eeab6f7d Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Thu, 28 Aug 2014 15:29:22 +0700
Subject: [PATCH 28/41] Clean up update_log function for readability

---
 src/couch_mrview_updater.erl | 109 +++++++++++++----------------------
 1 file changed, 39 insertions(+), 70 deletions(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 2a67dff..66db969 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -17,6 +17,8 @@
 -include_lib("couch/include/couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 
+-define(REM_VAL,  {[{<<"_removed">>, true}]}).
+
 
 start_update(Partial, State, NumChanges) ->
     QueueOpts = [{max_size, 100000}, {max_items, 500}],
@@ -367,83 +369,50 @@ update_id_btree(Btree, DocIdKeys, _) ->
     ToRem = [Id || {Id, DIKeys} <- DocIdKeys, DIKeys == []],
     couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem).
 
-walk_log(BTree, Fun, Acc, Ids) ->
-    WrapFun = fun(KV, _Offset, Acc2) ->
-            Fun(KV, Acc2)
-    end,
-    lists:foldl(fun(Id, Acc1) ->
-                Opt = [{start_key, Id}, {end_key, Id}],
-                {ok, _, A} = couch_btree:fold(BTree, WrapFun, Acc1, Opt),
-                A
-        end, Acc, Ids).
-
 update_log(Btree, Log, _UpdatedSeq, true) ->
-    ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log),
-                             DIKeys /= []],
+    ToAdd = lists:filter(fun({_, Keys}) -> Keys =/= [] end, dict:to_list(Log)),
     {ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
     {ok, dict:new(), dict:new(), LogBtree2};
 update_log(Btree, Log, UpdatedSeq, _) ->
     %% build list of updated keys and Id
-    {ToLook, Updated} = dict:fold(fun
-                (Id, [], {IdsAcc, KeysAcc}) ->
-                    {[Id | IdsAcc], KeysAcc};
-                (Id, DIKeys, {IdsAcc, KeysAcc}) ->
-                    KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}},
-                                               KeysAcc2) ->
-                                    [{Id, ViewId, Key} | KeysAcc2]
-                            end, KeysAcc, DIKeys),
-                {[Id | IdsAcc], KeysAcc1} end, {[], []}, Log),
-
-    RemValue = {[{<<"_removed">>, true}]},
-    {Log1, AddAcc, DelAcc} = walk_log(Btree, fun({DocId, VIdKeys},
-                                                          {Log2, AddAcc2, DelAcc2}) ->
-
-                {Log3, AddAcc3, DelAcc3} = lists:foldl(fun({ViewId,{Key, Seq, Op}},
-                                                           {Log4, AddAcc4, DelAcc4}) ->
-
-                            case lists:member({DocId, ViewId, Key}, Updated) of
-                                true ->
-                                    %% the log is updated, deleted old
-                                    %% record from the view
-                                    DelAcc5 = dict:append(ViewId,
-                                                          {Key, Seq, DocId},
-                                                          DelAcc4),
-                                    {Log4, AddAcc4, DelAcc5};
-                                false when Op /= del ->
-                                    %% an update operation has been
-                                    %% logged for this key. We must now
-                                    %% record it as deleted in the
-                                    %% log, remove the old record in
-                                    %% the view and update the view
-                                    %% with a removed record.
-                                    Log5 = dict:append(DocId,
-                                                       {ViewId,
-                                                        {Key,UpdatedSeq, del}},
-                                                       Log4),
-                                    DelAcc5 = dict:append(ViewId,
-                                                          {Key, Seq, DocId},
-                                                          DelAcc4),
-                                    AddAcc5 = dict:append(ViewId,
-                                                          {{UpdatedSeq, Key},
-                                                           {DocId, RemValue}},
-                                                          AddAcc4),
-                                    {Log5, AddAcc5, DelAcc5};
-                                false ->
-                                    %% the key has already been
-                                    %% registered in the view as
-                                    %% deleted, make sure to add it
-                                    %% to the new log.
-                                    Log5 = dict:append(DocId,
-                                                       {ViewId,
-                                                        {Key, Seq, del}}, Log4),
-                                    {Log5, AddAcc4, DelAcc4}
-                            end
-                    end, {Log2, AddAcc2, DelAcc2}, VIdKeys),
-                    {ok, {Log3, AddAcc3, DelAcc3}}
-            end, {Log, dict:new(), dict:new()}, ToLook),
+    {ToLook, Updated} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc}) ->
+        KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}}, KeysAcc2) ->
+            [{Id, ViewId, Key} | KeysAcc2]
+        end, KeysAcc, DIKeys),
+        {[Id | IdsAcc], KeysAcc1}
+    end, {[], []}, Log),
+
+    MapFun = fun({ok, KV}) -> [KV]; (not_found) -> [] end,
+    KVsToLook = lists:flatmap(MapFun, couch_btree:lookup(Btree, ToLook)),
+    {Log1, AddAcc, DelAcc} = lists:foldl(fun({DocId, VIdKeys}, Acc) ->
+        lists:foldl(fun({ViewId, {Key, Seq, Op}}, {Log4, AddAcc4, DelAcc4}) ->
+            case lists:member({DocId, ViewId, Key}, Updated) of
+                true ->
+                    % the log is updated, deleted old record from the view
+                    DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
+                    {Log4, AddAcc4, DelAcc5};
+                false when Op /= del ->
+                    % an update operation has been logged for this key. We must
+                    % now record it as deleted in the log, remove the old
+                    % record in the view and update the view with a removed
+                    % record.
+                    LogValue = {ViewId, {Key, UpdatedSeq, del}},
+                    Log5 = dict:append(DocId, LogValue, Log4),
+                    DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
+                    AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL}},
+                    AddAcc5 = dict:append(ViewId, AddValue, AddAcc4),
+                    {Log5, AddAcc5, DelAcc5};
+                false ->
+                    % the key has already been registered in the view as
+                    % deleted, make sure to add it to the new log.
+                    Log5 = dict:append(DocId, {ViewId, {Key, Seq, del}}, Log4),
+                    {Log5, AddAcc4, DelAcc4}
+            end
+        end, Acc, VIdKeys)
+    end, {Log, dict:new(), dict:new()}, KVsToLook),
 
     ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log1), DIKeys /= []],
-    %% store the new logs
+    % store the new logs
     {ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
     {ok, AddAcc, DelAcc, LogBtree2}.
 

From df3d28d0d25c6d7a4d053ed13c493e78d23735ba Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Thu, 28 Aug 2014 21:22:55 +0700
Subject: [PATCH 29/41] Fix removal of keys from view seq index

---
 src/couch_mrview_updater.erl | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 66db969..2dcce06 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -265,13 +265,17 @@ merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Log) ->
 
 
 merge_results({DocId, _Seq, Rev, []}, ViewKVs, DocIdKeys, Log) ->
-    {ViewKVs, [{DocId, Rev, []} | DocIdKeys], dict:store(DocId, [], Log)};
+    {ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
 merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Log) ->
     JsonResults = couch_query_servers:raw_to_ejson(RawResults),
     Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
-    {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [],
-                                            [], Log),
-    {ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}.
+    case Results of
+        [[],[]] ->
+            {ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
+        _ ->
+            {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [], [], Log),
+            {ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}
+    end.
 
 
 insert_results(DocId, _Seq, _Rev, [], [], ViewKVs, ViewIdKeys, Log) ->
@@ -285,7 +289,7 @@ insert_results(DocId, Seq, Rev, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVA
             {[{Key, {dups, [Val1, Val2]}} | Rest], IdKeys, Log2};
         ({Key, Value}, {Rest, IdKeys, Log2}) ->
             {[{Key, Value} | Rest], [{Id, Key} | IdKeys],
-             dict:append(DocId, {Id, {Key, Seq, add}}, Log2)}
+             dict:append({DocId, Rev}, {Id, {Key, Seq, add}}, Log2)}
     end,
     InitAcc = {[], VIdKeys, Log},
     couch_stats:increment_counter([couchdb, mrview, emits], length(KVs)),
@@ -375,12 +379,16 @@ update_log(Btree, Log, _UpdatedSeq, true) ->
     {ok, dict:new(), dict:new(), LogBtree2};
 update_log(Btree, Log, UpdatedSeq, _) ->
     %% build list of updated keys and Id
+    Revs = dict:from_list(dict:fetch_keys(Log)),
+    Log0 = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
+        dict:store(Id, DIKeys, Acc)
+    end, dict:new(), Log),
     {ToLook, Updated} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc}) ->
         KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}}, KeysAcc2) ->
             [{Id, ViewId, Key} | KeysAcc2]
         end, KeysAcc, DIKeys),
         {[Id | IdsAcc], KeysAcc1}
-    end, {[], []}, Log),
+    end, {[], []}, Log0),
 
     MapFun = fun({ok, KV}) -> [KV]; (not_found) -> [] end,
     KVsToLook = lists:flatmap(MapFun, couch_btree:lookup(Btree, ToLook)),
@@ -399,7 +407,8 @@ update_log(Btree, Log, UpdatedSeq, _) ->
                     LogValue = {ViewId, {Key, UpdatedSeq, del}},
                     Log5 = dict:append(DocId, LogValue, Log4),
                     DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
-                    AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL}},
+                    Rev = dict:fetch(DocId, Revs),
+                    AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL, Rev}},
                     AddAcc5 = dict:append(ViewId, AddValue, AddAcc4),
                     {Log5, AddAcc5, DelAcc5};
                 false ->
@@ -409,7 +418,7 @@ update_log(Btree, Log, UpdatedSeq, _) ->
                     {Log5, AddAcc4, DelAcc4}
             end
         end, Acc, VIdKeys)
-    end, {Log, dict:new(), dict:new()}, KVsToLook),
+    end, {Log0, dict:new(), dict:new()}, KVsToLook),
 
     ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log1), DIKeys /= []],
     % store the new logs

From 487df67528a14414b2048b4648b0d650a474945d Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Thu, 28 Aug 2014 23:40:59 +0700
Subject: [PATCH 30/41] Fix stuff which I broke in a rebase. I'll fix the git
 history later

---
 src/couch_mrview_index.erl | 14 +++++++++-----
 src/couch_mrview_util.erl  | 13 +++----------
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 63340cb..893ecd1 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -64,10 +64,14 @@ get(Property, State) ->
                 design_opts = Opts
             } = State,
             {ok, FileSize} = couch_file:bytes(Fd),
-            {ok, ExternalSize} = couch_mrview_util:calculate_external_size(IdBtree,
-                                                                           LogBtree,
-                                                                           Views),
-            ActiveSize = ExternalSize + couch_btree:size(Btree),
+            {ok, ExternalSize} = couch_mrview_util:calculate_external_size(Views),
+            LogBtSize = case LogBtree of
+                nil ->
+                    0;
+                _ ->
+                    couch_btree:size(LogBtree)
+            end,
+            ActiveSize = couch_btree:size(IdBtree) + LogBtSize + ExternalSize,
 
             IncDesign = couch_util:get_value(<<"include_design">>, Opts, false),
             LocalSeq = couch_util:get_value(<<"local_seq">>, Opts, false),
@@ -183,7 +187,7 @@ finish_update(State) ->
 commit(State) ->
     Header = {State#mrst.sig, couch_mrview_util:make_header(State)},
     Resp = couch_file:write_header(State#mrst.fd, Header),
-    couch_mrview_update_notifier:notify({index_update, State#mrst.db_name, State#mrst.idx_name}),
+    couch_event:notify(State#mrst.db_name, {index_update, State#mrst.idx_name}),
     Resp.
 
 
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index ce037d5..2850718 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -22,7 +22,7 @@
 -export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]).
 -export([fold/4, fold_reduce/4]).
 -export([temp_view_to_ddoc/1]).
--export([calculate_external_size/3]).
+-export([calculate_external_size/1]).
 -export([validate_args/1]).
 -export([maybe_load_doc/3, maybe_load_doc/4]).
 -export([maybe_update_index_file/1]).
@@ -791,7 +791,7 @@ changes_ekey_opts(_StartSeq, #mrargs{end_key=EKey,
 
 
 
-calculate_external_size(IdBt, LogBt, Views) ->
+calculate_external_size(Views) ->
     SumFun = fun(#mrview{btree=Bt, seq_btree=SBt, key_byseq_btree=KSBt}, Acc) ->
         Size0 = sum_btree_sizes(Acc, couch_btree:size(Bt)),
         Size1 = case SBt of
@@ -803,14 +803,7 @@ calculate_external_size(IdBt, LogBt, Views) ->
             _ -> sum_btree_sizes(Size1, couch_btree:size(KSBt))
         end
     end,
-    Size = case LogBt of
-        nil ->
-            lists:foldl(SumFun, couch_btree:size(IdBt), Views);
-        _ ->
-            lists:foldl(SumFun, couch_btree:size(IdBt) +
-                        couch_btree:size(LogBt), Views)
-    end,
-    {ok, Size}.
+    {ok, lists:foldl(SumFun, 0, Views)}.
 
 
 sum_btree_sizes(nil, _) ->

From d6771df75c68d0796d4758e02b653a2d211707e3 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Mon, 8 Sep 2014 15:10:17 -0700
Subject: [PATCH 31/41] Port view changes tests to eunit

---
 src/couch_mrview_test_util.erl                |  2 +-
 test/09-index-events.t                        | 73 --------------
 ...t => couch_mrview_changes_since_tests.erl} | 98 ++++++++++---------
 ...t => couch_mrview_index_changes_tests.erl} | 69 ++++++-------
 4 files changed, 90 insertions(+), 152 deletions(-)
 delete mode 100644 test/09-index-events.t
 rename test/{08-changes_since.t => couch_mrview_changes_since_tests.erl} (70%)
 rename test/{10-index-changes.t => couch_mrview_index_changes_tests.erl} (82%)

diff --git a/src/couch_mrview_test_util.erl b/src/couch_mrview_test_util.erl
index 1d3d788..c8da1ee 100644
--- a/src/couch_mrview_test_util.erl
+++ b/src/couch_mrview_test_util.erl
@@ -34,7 +34,7 @@ new_db(Name, Type) ->
     save_docs(Db, [ddoc(Type)]).
 
 delete_db(Name) ->
-    couch_server:delete(Name, [{user_ctx, ?ADMIN}]).
+    couch_server:delete(Name, [?ADMIN_USER]).
 
 save_docs(Db, Docs) ->
     {ok, _} = couch_db:update_docs(Db, Docs, []),
diff --git a/test/09-index-events.t b/test/09-index-events.t
deleted file mode 100644
index 26d904d..0000000
--- a/test/09-index-events.t
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-%   http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-main(_) ->
-    etap:plan(5),
-    case (catch test()) of
-        ok ->
-            etap:end_tests();
-        Other ->
-            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
-            etap:bail(Other)
-    end,
-    timer:sleep(300),
-    ok.
-
-test() ->
-    test_util:start_couch(),
-    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
-    test_info(Db),
-    test_update_event(Db),
-    test_delete_event(Db),
-    test_util:stop_couch(),
-    ok.
-
-test_info(Db) ->
-    {ok, Info} = couch_mrview:get_info(Db, <<"_design/bar">>),
-    etap:is(getval(update_options, Info), [<<"seq_indexed">>],
-            "update options OK"),
-    ok.
-
-test_update_event(Db) ->
-    {ok, Pid} = couch_index_event:start_link(self()),
-    etap:ok(is_pid(Pid), "event handler added"),
-    ok = couch_mrview:refresh(Db, <<"_design/bar">>),
-    Expect = {index_update, {<<"foo">>, <<"_design/bar">>,
-                             couch_mrview_index}},
-    receive
-        Event ->
-            etap:is(Event, Expect, "index update events OK")
-    end,
-    couch_index_event:stop(Pid).
-
-test_delete_event(Db) ->
-    ok = couch_mrview:refresh(Db, <<"_design/bar">>),
-    timer:sleep(300),
-    {ok, Pid} = couch_index_event:start_link(self()),
-    etap:ok(is_pid(Pid), "delete event handler added"),
-
-
-    catch couch_mrview_test_util:delete_db(<<"foo">>),
-    Expect = {index_delete, {<<"foo">>, <<"_design/bar">>,
-                             couch_mrview_index}},
-    receive
-        Event ->
-            etap:is(Event, Expect, "index delete events OK")
-    end,
-    couch_index_event:stop(Pid).
-
-getval(Key, PL) ->
-    {value, {Key, Val}} = lists:keysearch(Key, 1, PL),
-    Val.
diff --git a/test/08-changes_since.t b/test/couch_mrview_changes_since_tests.erl
similarity index 70%
rename from test/08-changes_since.t
rename to test/couch_mrview_changes_since_tests.erl
index 3107c1a..65f5a0b 100644
--- a/test/08-changes_since.t
+++ b/test/couch_mrview_changes_since_tests.erl
@@ -1,6 +1,3 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-
 % Licensed under the Apache License, Version 2.0 (the "License"); you may not
 % use this file except in compliance with the License. You may obtain a copy of
 % the License at
@@ -13,37 +10,50 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
-main(_) ->
-    etap:plan(14),
-    case (catch test()) of
-        ok ->
-            etap:end_tests();
-        Other ->
-            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
-            etap:bail(Other)
-    end,
-    timer:sleep(300),
-    ok.
+-module(couch_mrview_changes_since_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(TIMEOUT, 1000).
 
-test() ->
-    test_util:start_couch(),
-
-    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
-
-    test_basic(Db),
-    test_range(Db),
-    test_basic_since(Db),
-    test_range_since(Db),
-    test_basic_count(Db),
-    test_range_count(Db),
-    test_basic_count_since(Db),
-    test_range_count_since(Db),
-    test_compact(Db),
-    test_remove_key(Db),
-    catch test_util:stop_couch(),
+
+
+setup() ->
+    {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), changes),
+    Db.
+
+teardown(Db) ->
+    couch_db:close(Db),
+    couch_server:delete(Db#db.name, [?ADMIN_USER]),
     ok.
 
 
+changes_since_test() ->
+    {
+        "changes_since tests",
+        {
+            setup,
+            fun test_util:start_couch/0, fun test_util:stop_couch/1,
+            {
+                foreach,
+                fun setup/0, fun teardown/1,
+                [
+                    test_basic/1,
+                    test_range/1,
+                    test_basic_since/1,
+                    test_range_since/1,
+                    test_basic_count/1,
+                    test_range_count/1,
+                    test_basic_count_since/1,
+                    test_range_count_since/1,
+                    test_compact/1,
+                    test_remove_key/1
+                ]
+            }
+        }
+    }.
+
 test_basic(Db) ->
     Result = run_query(Db, 0, []),
     Expect = {ok, [
@@ -58,7 +68,7 @@ test_basic(Db) ->
                 {{10, 8, <<"8">>}, 8},
                 {{11, 9, <<"9">>}, 9}
     ]},
-    etap:is(Result, Expect, "Simple view query worked.").
+    ?_assertEqual(Result, Expect).
 
 
 test_range(Db) ->
@@ -68,7 +78,7 @@ test_range(Db) ->
                 {{6, 4, <<"4">>}, 4},
                 {{7, 5, <<"5">>}, 5}
     ]},
-    etap:is(Result, Expect, "Query with range works.").
+    ?_assertEqual(Result, Expect).
 
 test_basic_since(Db) ->
     Result = run_query(Db, 5, []),
@@ -80,7 +90,7 @@ test_basic_since(Db) ->
                 {{10, 8, <<"8">>}, 8},
                 {{11, 9, <<"9">>}, 9}
     ]},
-    etap:is(Result, Expect, "Simple view query since 5 worked.").
+    ?_assertEqual(Result, Expect).
 
 test_range_since(Db) ->
     Result = run_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
@@ -88,29 +98,29 @@ test_range_since(Db) ->
                 {{6, 4, <<"4">>}, 4},
                 {{7, 5, <<"5">>}, 5}
     ]},
-    etap:is(Result, Expect, "Query with range since 5 works.").
+    ?_assertEqual(Result, Expect).
 
 test_basic_count(Db) ->
     Result = run_count_query(Db, 0, []),
-    etap:is(Result, 10, "Simple view count worked.").
+    ?_assertEqual(Result, 10).
 
 test_range_count(Db) ->
     Result = run_count_query(Db, 0, [{start_key, 3}, {end_key, 5}]),
-    etap:is(Result, 3, "Count with range works.").
+    ?_assertEqual(Result, 3).
 
 test_basic_count_since(Db) ->
     Result = run_count_query(Db, 5, []),
-    etap:is(Result, 6, "Simple view count since 5 worked.").
+    ?_assertEqual(Result, 6).
 
 test_range_count_since(Db) ->
     Result = run_count_query(Db, 5, [{start_key, 3}, {end_key, 5}]),
-    etap:is(Result, 2, "Count with range since 5 works.").
+    ?_assertEqual(Result, 2).
 
 test_compact(Db) ->
     Result = couch_mrview:compact(Db, <<"_design/bar">>),
-    etap:is(Result, ok, "compact view is OK"),
+    ?_assertEqual(Result, ok),
     Count = run_count_query(Db, 0, []),
-    etap:is(Count, 10, "compact view worked.").
+    ?_assertEqual(Count, 10).
 
 test_remove_key(Db) ->
     %% add new doc
@@ -120,13 +130,13 @@ test_remove_key(Db) ->
     {ok, _} =  couch_db:ensure_full_commit(Db),
     {ok, Db1} = couch_db:reopen(Db),
     Result = run_count_query(Db1, 0, []),
-    etap:is(Result, 11, "Add new doc worked."),
+    ?_assertEqual(Result, 11),
     %% check new view key
     Result1 = run_query(Db1, 0, [{start_key, 11}, {end_key, 11}]),
     Expect = {ok, [
                 {{12, 11, <<"11">>}, 11}
     ]},
-    etap:is(Result1, Expect, "added key OK."),
+    ?_assertEqual(Result1, Expect),
 
     %% delete doc
     Doc2 = couch_doc:from_json_obj({[
@@ -137,13 +147,13 @@ test_remove_key(Db) ->
     {ok, _} = couch_db:update_doc(Db1, Doc2, []),
     {ok, Db2} = couch_db:reopen(Db1),
     Result2 = run_count_query(Db2, 0, []),
-    etap:is(Result2, 11, "removed key saved."),
+    ?_assertEqual(Result2, 11),
     %% check new view key
     Result3 = run_query(Db2, 0, [{start_key, 11}, {end_key, 11}]),
     Expect2 = {ok, [
                 {{13, 11, <<"11">>}, {[{<<"_removed">>, true}]}}
     ]},
-    etap:is(Result3, Expect2, "removed key OK.").
+    ?_assertEqual(Result3, Expect2).
 
 run_query(Db, Since, Opts) ->
     Fun = fun(KV, Acc) -> {ok, [KV | Acc]} end,
diff --git a/test/10-index-changes.t b/test/couch_mrview_index_changes_tests.erl
similarity index 82%
rename from test/10-index-changes.t
rename to test/couch_mrview_index_changes_tests.erl
index 0c5e924..4a3a35c 100644
--- a/test/10-index-changes.t
+++ b/test/couch_mrview_index_changes_tests.erl
@@ -1,6 +1,3 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-
 % Licensed under the Apache License, Version 2.0 (the "License"); you may not
 % use this file except in compliance with the License. You may obtain a copy of
 % the License at
@@ -13,30 +10,34 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
-main(_) ->
-    etap:plan(8),
-    case (catch test()) of
-        ok ->
-            etap:end_tests();
-        Other ->
-            etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
-            etap:bail(Other)
-    end,
-    timer:sleep(300),
-    ok.
+-module(couch_mrview_all_docs_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+
+changes_index_test() ->
+    {
+        "changes index tests",
+        {
+            setup,
+            fun test_util:start_couch/0, fun test_util:stop_couch/1,
+            {
+                foreach,
+                fun setup/0, fun teardown/1,
+                [
+                    fun test_normal_changes/1,
+                    fun test_stream_once/1,
+                    fun test_stream_once_since/1,
+                    fun test_stream_once_timeout/1,
+                    fun test_stream_once_heartbeat/1,
+                    fun test_stream/1,
+                    fun test_indexer/1
+                ]
+            }
+        }
+    }.
 
-test() ->
-    test_util:start_couch(),
-    {ok, Db} = couch_mrview_test_util:init_db(<<"foo">>, changes),
-    test_normal_changes(Db),
-    test_stream_once(Db),
-    test_stream_once_since(Db),
-    test_stream_once_timeout(Db),
-    test_stream_once_heartbeat(Db),
-    test_stream(Db),
-    test_indexer(Db),
-    test_util:stop_couch(),
-    ok.
 
 test_normal_changes(Db) ->
     Result = run_query(Db, []),
@@ -52,7 +53,7 @@ test_normal_changes(Db) ->
                 {{10, 8, <<"8">>}, 8},
                 {{11, 9, <<"9">>}, 9}
     ]},
-    etap:is(Result, Expect, "normal changes worked.").
+    ?_assertEqual(Result, Expect).
 
 test_stream_once(Db) ->
     Result = run_query(Db, [{stream, once}]),
@@ -68,7 +69,7 @@ test_stream_once(Db) ->
                 {{10, 8, <<"8">>}, 8},
                 {{11, 9, <<"9">>}, 9}
     ]},
-    etap:is(Result, Expect, "stream once since 0 worked.").
+    ?_assertEqual(Result, Expect).
 
 
 test_stream_once_since(Db) ->
@@ -89,7 +90,7 @@ test_stream_once_since(Db) ->
 
     receive
         {result, Result} ->
-            etap:is(Result, Expect, "normal changes worked.")
+            ?_assertEqual(Result, Expect)
     after 5000 ->
             io:format("never got the change", [])
     end.
@@ -110,7 +111,7 @@ test_stream_once_timeout(Db) ->
 
     receive
         {result, Result} ->
-            etap:is(Result, Expect, "got timeout.")
+            ?_assertEqual(Result, Expect)
     after 5000 ->
             io:format("never got the change", [])
     end.
@@ -139,7 +140,7 @@ test_stream_once_heartbeat(Db) ->
 
     receive
         {result, Result} ->
-            etap:is(Result, Expect, "heartbeat OK.")
+            ?_assertEqual(Result, Expect)
     after 5000 ->
             io:format("never got the change", [])
     end.
@@ -167,7 +168,7 @@ test_stream(Db) ->
 
     receive
         {result, Result} ->
-            etap:is(Result, Expect, "stream OK.")
+            ?_assertEqual(Result, Expect)
     after 5000 ->
             io:format("never got the change", [])
     end.
@@ -176,14 +177,14 @@ test_stream(Db) ->
 test_indexer(Db) ->
     Result = run_query(Db, [{since, 14}, refresh]),
     Expect = {ok, 15, [{{15,14,<<"14">>},14}]},
-    etap:is(Result, Expect, "refresh index by hand OK."),
+    ?_assertEqual(Result, Expect),
 
     {ok, Db1} = save_doc(Db, 15),
     timer:sleep(1500),
     Result1 = run_query(Db1, [{since, 14}], false),
     Expect1 = {ok, 16, [{{15,14,<<"14">>},14},
                        {{16,15,<<"15">>},15}]},
-    etap:is(Result1, Expect1, "changes indexed in background OK."),
+    ?_assertEqual(Result1, Expect1),
     ok.
 
 

From fa7a888f52441355f3156514d113e2fcbb3750e6 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Mon, 8 Sep 2014 16:26:04 -0700
Subject: [PATCH 32/41] Make view sig backwards compatible

---
 src/couch_mrview_util.erl | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 2850718..6054e3a 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -162,9 +162,12 @@ view_sig(Db, State, View, #mrargs{include_docs=true}=Args) ->
     BaseSig = view_sig(Db, State, View, Args#mrargs{include_docs=false}),
     UpdateSeq = couch_db:get_update_seq(Db),
     PurgeSeq = couch_db:get_purge_seq(Db),
-    Bin = term_to_binary({BaseSig, UpdateSeq, PurgeSeq,
-                          State#mrst.seq_indexed, State#mrst.keyseq_indexed}),
-    couch_index_util:hexsig(couch_util:md5(Bin));
+    #mrst{
+        seq_indexed=SeqIndexed,
+        keyseq_indexed=KeySeqIndexed
+    } = State,
+    Term = view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed),
+    couch_index_util:hexsig(couch_util:md5(term_to_binary(Term)));
 view_sig(Db, State, {_Nth, _Lang, View}, Args) ->
     view_sig(Db, State, View, Args);
 view_sig(_Db, State, View, Args0) ->
@@ -177,8 +180,18 @@ view_sig(_Db, State, View, Args0) ->
         preflight_fun=undefined,
         extra=[]
     },
-    Bin = term_to_binary({Sig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args}),
-    couch_index_util:hexsig(couch_util:md5(Bin)).
+    Term = view_sig_term(Sig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args),
+    couch_index_util:hexsig(couch_util:md5(term_to_binary(Term))).
+
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false) ->
+    {BaseSig, UpdateSeq, PurgeSeq};
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed) ->
+    {BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed}.
+
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, false, false, Args) ->
+    {BaseSig, UpdateSeq, PurgeSeq, Args};
+view_sig_term(BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args) ->
+    {BaseSig, UpdateSeq, PurgeSeq, KeySeqIndexed, SeqIndexed, Args}.
 
 
 init_state(Db, Fd, #mrst{views=Views}=State, nil) ->

From 5eab8107b6671bc8a102bbb262ebb49ca442d7f0 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Tue, 9 Sep 2014 09:34:21 -0700
Subject: [PATCH 33/41] Fix process_doc for temp views

---
 src/couch_mrview_updater.erl | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 2dcce06..1cfa9c0 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -138,12 +138,16 @@ process_doc(Doc, Seq, #mrst{doc_acc=Acc}=State) when length(Acc) > 100 ->
 process_doc(nil, Seq, #mrst{doc_acc=Acc}=State) ->
     {ok, State#mrst{doc_acc=[{nil, Seq, nil, nil} | Acc]}};
 process_doc(#doc{id=Id, deleted=true}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
-    {RevPos, [Rev | _]} = Doc#doc.revs,
-    {ok, State#mrst{doc_acc=[{Id, Seq, {RevPos, Rev}, deleted} | Acc]}};
+    Rev= extract_rev(Doc#doc.revs),
+    {ok, State#mrst{doc_acc=[{Id, Seq, Rev, deleted} | Acc]}};
 process_doc(#doc{id=Id}=Doc, Seq, #mrst{doc_acc=Acc}=State) ->
-    {RevPos, [Rev | _]} = Doc#doc.revs,
-    {ok, State#mrst{doc_acc=[{Id, Seq, {RevPos, Rev}, Doc} | Acc]}}.
+    Rev = extract_rev(Doc#doc.revs),
+    {ok, State#mrst{doc_acc=[{Id, Seq, Rev, Doc} | Acc]}}.
 
+extract_rev({0, []}) ->
+    {0, []};
+extract_rev({RevPos, [Rev | _]}) ->
+    {RevPos, Rev}.
 
 finish_update(#mrst{doc_acc=Acc}=State) ->
     if Acc /= [] ->

From 8d3cdf162db47ce87baa2238f658f8fe05724051 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Tue, 9 Sep 2014 09:52:49 -0700
Subject: [PATCH 34/41] fix broken tests

---
 test/couch_mrview_changes_since_tests.erl | 20 ++++++++++----------
 test/couch_mrview_index_changes_tests.erl | 11 ++++++++++-
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/test/couch_mrview_changes_since_tests.erl b/test/couch_mrview_changes_since_tests.erl
index 65f5a0b..69c9646 100644
--- a/test/couch_mrview_changes_since_tests.erl
+++ b/test/couch_mrview_changes_since_tests.erl
@@ -39,16 +39,16 @@ changes_since_test() ->
                 foreach,
                 fun setup/0, fun teardown/1,
                 [
-                    test_basic/1,
-                    test_range/1,
-                    test_basic_since/1,
-                    test_range_since/1,
-                    test_basic_count/1,
-                    test_range_count/1,
-                    test_basic_count_since/1,
-                    test_range_count_since/1,
-                    test_compact/1,
-                    test_remove_key/1
+                    fun test_basic/1,
+                    fun test_range/1,
+                    fun test_basic_since/1,
+                    fun test_range_since/1,
+                    fun test_basic_count/1,
+                    fun test_range_count/1,
+                    fun test_basic_count_since/1,
+                    fun test_range_count_since/1,
+                    fun test_compact/1,
+                    fun test_remove_key/1
                 ]
             }
         }
diff --git a/test/couch_mrview_index_changes_tests.erl b/test/couch_mrview_index_changes_tests.erl
index 4a3a35c..486deb5 100644
--- a/test/couch_mrview_index_changes_tests.erl
+++ b/test/couch_mrview_index_changes_tests.erl
@@ -10,12 +10,21 @@
 % License for the specific language governing permissions and limitations under
 % the License.
 
--module(couch_mrview_all_docs_tests).
+-module(couch_mrview_index_changes_tests).
 
 -include_lib("couch/include/couch_eunit.hrl").
 -include_lib("couch/include/couch_db.hrl").
 
 
+setup() ->
+    {ok, Db} = couch_mrview_test_util:init_db(?tempdb(), map),
+    Db.
+
+teardown(Db) ->
+    couch_db:close(Db),
+    couch_server:delete(Db#db.name, [?ADMIN_USER]),
+    ok.
+
 changes_index_test() ->
     {
         "changes index tests",

From b01a0bf2d08ff357493f90effea8e42581db475b Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Thu, 11 Sep 2014 12:21:45 -0700
Subject: [PATCH 35/41] fix first-pass log key addition and seq tree row
 removal

---
 src/couch_mrview_updater.erl | 52 ++++++++++++++++++++++--------------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index 1cfa9c0..f92a2a2 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -378,7 +378,14 @@ update_id_btree(Btree, DocIdKeys, _) ->
     couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem).
 
 update_log(Btree, Log, _UpdatedSeq, true) ->
-    ToAdd = lists:filter(fun({_, Keys}) -> Keys =/= [] end, dict:to_list(Log)),
+    ToAdd = lists:flatmap(fun({{Id, _Rev}, Keys}) ->
+        case Keys of
+            [] ->
+                [];
+            _ ->
+                [{Id, Keys}]
+        end
+    end, dict:to_list(Log)),
     {ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
     {ok, dict:new(), dict:new(), LogBtree2};
 update_log(Btree, Log, UpdatedSeq, _) ->
@@ -387,39 +394,44 @@ update_log(Btree, Log, UpdatedSeq, _) ->
     Log0 = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
         dict:store(Id, DIKeys, Acc)
     end, dict:new(), Log),
-    {ToLook, Updated} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc}) ->
-        KeysAcc1 = lists:foldl(fun({ViewId, {Key, _Seq, _Op}}, KeysAcc2) ->
-            [{Id, ViewId, Key} | KeysAcc2]
-        end, KeysAcc, DIKeys),
-        {[Id | IdsAcc], KeysAcc1}
-    end, {[], []}, Log0),
+    {ToLook, Updated, Removed} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc, RemAcc}) ->
+        {KeysAcc1, RemAcc1} = lists:foldl(fun({ViewId, {Key, _Seq, Op}}, Acc) ->
+            {KeysAcc2, RemAcc2} = Acc,
+            case Op of
+                add ->
+                    {[{Id, ViewId, Key} | KeysAcc2], RemAcc2};
+                del ->
+                    {KeysAcc2, [{Id, ViewId, Key} | RemAcc2]}
+            end
+        end, {KeysAcc, RemAcc}, DIKeys),
+        {[Id | IdsAcc], KeysAcc1, RemAcc1}
+    end, {[], [], []}, Log0),
 
     MapFun = fun({ok, KV}) -> [KV]; (not_found) -> [] end,
     KVsToLook = lists:flatmap(MapFun, couch_btree:lookup(Btree, ToLook)),
     {Log1, AddAcc, DelAcc} = lists:foldl(fun({DocId, VIdKeys}, Acc) ->
-        lists:foldl(fun({ViewId, {Key, Seq, Op}}, {Log4, AddAcc4, DelAcc4}) ->
+        lists:foldl(fun({ViewId, {Key, Seq, _Op}}, {Log4, AddAcc4, DelAcc4}) ->
             case lists:member({DocId, ViewId, Key}, Updated) of
                 true ->
                     % the log is updated, deleted old record from the view
                     DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
                     {Log4, AddAcc4, DelAcc5};
-                false when Op /= del ->
+                false ->
                     % an update operation has been logged for this key. We must
-                    % now record it as deleted in the log, remove the old
-                    % record in the view and update the view with a removed
-                    % record.
-                    LogValue = {ViewId, {Key, UpdatedSeq, del}},
-                    Log5 = dict:append(DocId, LogValue, Log4),
+                    % now record it as deleted in the log, remove the old record
+                    % in the view and update the view with a removed record.
+                    Log5 = case lists:member({DocId, ViewId, Key}, Removed) of
+                        false ->
+                            LogValue = {ViewId, {Key, UpdatedSeq, del}},
+                            dict:append(DocId, LogValue, Log4);
+                        true ->
+                            Log4
+                    end,
                     DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
                     Rev = dict:fetch(DocId, Revs),
                     AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL, Rev}},
                     AddAcc5 = dict:append(ViewId, AddValue, AddAcc4),
-                    {Log5, AddAcc5, DelAcc5};
-                false ->
-                    % the key has already been registered in the view as
-                    % deleted, make sure to add it to the new log.
-                    Log5 = dict:append(DocId, {ViewId, {Key, Seq, del}}, Log4),
-                    {Log5, AddAcc4, DelAcc4}
+                    {Log5, AddAcc5, DelAcc5}
             end
         end, Acc, VIdKeys)
     end, {Log0, dict:new(), dict:new()}, KVsToLook),

From 2e0be3567c68968b24f1a233fb7b5e399e961ea8 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Tue, 16 Sep 2014 12:27:26 -0700
Subject: [PATCH 36/41] make REM_VAL an atom in order to differentiate between
 possible emitted values and the key-removed placeholder.

---
 src/couch_mrview_updater.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index f92a2a2..b7f91e1 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -17,7 +17,7 @@
 -include_lib("couch/include/couch_db.hrl").
 -include_lib("couch_mrview/include/couch_mrview.hrl").
 
--define(REM_VAL,  {[{<<"_removed">>, true}]}).
+-define(REM_VAL, removed).
 
 
 start_update(Partial, State, NumChanges) ->

From 11d4c17e4ca729832396d8b393b8c0b95bad7bc0 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Tue, 16 Sep 2014 12:52:54 -0700
Subject: [PATCH 37/41] fix removal of duplicate keys from seq tree

---
 src/couch_mrview_updater.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index b7f91e1..c98c2f5 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -273,8 +273,8 @@ merge_results({DocId, _Seq, Rev, []}, ViewKVs, DocIdKeys, Log) ->
 merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Log) ->
     JsonResults = couch_query_servers:raw_to_ejson(RawResults),
     Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
-    case Results of
-        [[],[]] ->
+    case lists:flatten(Results) of
+        [] ->
             {ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
         _ ->
             {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [], [], Log),

From eb05dd782437eaea32f198d78ae10bbccae2edd4 Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Fri, 19 Sep 2014 03:41:56 -0700
Subject: [PATCH 38/41] Fix seqs in mrview updater

This commit makes it so the correct seqs are inserted into the mrview
sequence tree. It also fixes the btree comparison function for that
tree.
---
 src/couch_mrview_updater.erl | 119 +++++++++++++++++------------------
 src/couch_mrview_util.erl    |   3 +-
 2 files changed, 60 insertions(+), 62 deletions(-)

diff --git a/src/couch_mrview_updater.erl b/src/couch_mrview_updater.erl
index c98c2f5..db5b5e9 100644
--- a/src/couch_mrview_updater.erl
+++ b/src/couch_mrview_updater.erl
@@ -206,8 +206,8 @@ write_results(Parent, State) ->
     case accumulate_writes(State, State#mrst.write_queue, nil) of
         stop ->
             Parent ! {new_state, State};
-        {Go, {Seq, ViewKVs, DocIdKeys, Log}} ->
-            NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys, Log),
+        {Go, {Seq, ViewKVs, DocIdKeys, Seqs, Log}} ->
+            NewState = write_kvs(State, Seq, ViewKVs, DocIdKeys, Seqs, Log),
             if Go == stop ->
                 Parent ! {new_state, NewState};
             true ->
@@ -229,17 +229,17 @@ start_query_server(State) ->
 
 
 accumulate_writes(State, W, Acc0) ->
-    {Seq, ViewKVs, DocIdKVs, Log} = case Acc0 of
-        nil -> {0, [{V#mrview.id_num, {[], []}} || V <- State#mrst.views], [], dict:new()};
+    {Seq, ViewKVs, DocIdKVs, Seqs, Log} = case Acc0 of
+        nil -> {0, [{V#mrview.id_num, {[], []}} || V <- State#mrst.views], [], dict:new(), dict:new()};
         _ -> Acc0
     end,
     case couch_work_queue:dequeue(W) of
         closed when Seq == 0 ->
             stop;
         closed ->
-            {stop, {Seq, ViewKVs, DocIdKVs, Log}};
+            {stop, {Seq, ViewKVs, DocIdKVs, Seqs, Log}};
         {ok, Info} ->
-            {_, _, NewIds, _} = Acc = merge_results(Info, Seq, ViewKVs, DocIdKVs, Log),
+            {_, _, NewIds, _, _} = Acc = merge_results(Info, Seq, ViewKVs, DocIdKVs, Seqs, Log),
             case accumulate_more(length(NewIds)) of
                 true -> accumulate_writes(State, W, Acc);
                 false -> {ok, Acc}
@@ -256,29 +256,27 @@ accumulate_more(NumDocIds) ->
         andalso CurrMem < list_to_integer(MinSize).
 
 
-merge_results([], SeqAcc, ViewKVs, DocIdKeys, Log) ->
-    {SeqAcc, ViewKVs, DocIdKeys, Log};
-merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Log) ->
-    Fun = fun(RawResults, {VKV, DIK, Log2}) ->
-        merge_results(RawResults, VKV, DIK, Log2)
+merge_results([], SeqAcc, ViewKVs, DocIdKeys, Seqs, Log) ->
+    {SeqAcc, ViewKVs, DocIdKeys, Seqs, Log};
+merge_results([{Seq, Results} | Rest], SeqAcc, ViewKVs, DocIdKeys, Seqs, Log) ->
+    Fun = fun(RawResults, {VKV, DIK, Seqs2, Log2}) ->
+        merge_results(RawResults, VKV, DIK, Seqs2, Log2)
     end,
-    {ViewKVs1, DocIdKeys1, Log1} = lists:foldl(Fun, {ViewKVs, DocIdKeys, Log},
-                                               Results),
-    merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1,
-                  Log1).
+    {ViewKVs1, DocIdKeys1, Seqs1, Log1} = lists:foldl(Fun, {ViewKVs, DocIdKeys, Seqs, Log}, Results),
+    merge_results(Rest, erlang:max(Seq, SeqAcc), ViewKVs1, DocIdKeys1, Seqs1, Log1).
 
 
-merge_results({DocId, _Seq, Rev, []}, ViewKVs, DocIdKeys, Log) ->
-    {ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
-merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Log) ->
+merge_results({DocId, Seq, Rev, []}, ViewKVs, DocIdKeys, Seqs, Log) ->
+    {ViewKVs, [{DocId, []} | DocIdKeys], dict:store(DocId, Seq, Seqs), dict:store({DocId, Rev}, [], Log)};
+merge_results({DocId, Seq, Rev, RawResults}, ViewKVs, DocIdKeys, Seqs, Log) ->
     JsonResults = couch_query_servers:raw_to_ejson(RawResults),
     Results = [[list_to_tuple(Res) || Res <- FunRs] || FunRs <- JsonResults],
     case lists:flatten(Results) of
         [] ->
-            {ViewKVs, [{DocId, []} | DocIdKeys], dict:store({DocId, Rev}, [], Log)};
+            {ViewKVs, [{DocId, []} | DocIdKeys], dict:store(DocId, Seq, Seqs), dict:store({DocId, Rev}, [], Log)};
         _ ->
             {ViewKVs1, ViewIdKeys, Log1} = insert_results(DocId, Seq, Rev, Results, ViewKVs, [], [], Log),
-            {ViewKVs1, [ViewIdKeys | DocIdKeys], Log1}
+            {ViewKVs1, [ViewIdKeys | DocIdKeys], dict:store(DocId, Seq, Seqs), Log1}
     end.
 
 
@@ -305,19 +303,25 @@ insert_results(DocId, Seq, Rev, [KVs | RKVs], [{Id, {VKVs, SKVs}} | RVKVs], VKVA
                   [{Id, {FinalKVs, FinalSKVs}} | VKVAcc], VIdKeys0, Log1).
 
 
-write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
+write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Seqs, Log0) ->
     #mrst{
         id_btree=IdBtree,
         log_btree=LogBtree,
         first_build=FirstBuild
     } = State,
 
+    Revs = dict:from_list(dict:fetch_keys(Log0)),
+
+    Log = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
+        dict:store(Id, DIKeys, Acc)
+    end, dict:new(), Log0),
+
     {ok, ToRemove, IdBtree2} = update_id_btree(IdBtree, DocIdKeys, FirstBuild),
     ToRemByView = collapse_rem_keys(ToRemove, dict:new()),
 
     {ok, SeqsToAdd, SeqsToRemove, LogBtree2} = case LogBtree of
         nil -> {ok, undefined, undefined, nil};
-        _ -> update_log(LogBtree, Log, UpdateSeq, FirstBuild)
+        _ -> update_log(LogBtree, Log, Revs, Seqs, FirstBuild)
     end,
 
     UpdateView = fun(#mrview{id_num=ViewId}=View, {ViewId, {KVs, SKVs}}) ->
@@ -336,12 +340,13 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
             SKVs1 = SKVs ++ SToAdd,
 
             {ok, SeqBtree2} = if SIndexed ->
-                RemSKs = [{Seq, Key} || {Key, Seq, _} <- SToRem],
+                RemSKs = lists:sort([{Seq, Key} || {Key, Seq, _} <- SToRem]),
                 couch_btree:add_remove(View#mrview.seq_btree,
                                        SKVs1, RemSKs);
             true ->
                 {ok, nil}
             end,
+
             {ok, KeyBySeqBtree2} = if KSIndexed ->
                 RemKSs = [{[Key, Seq], DocId} || {Key, Seq, DocId} <- SToRem],
                 couch_btree:add_remove(View#mrview.key_byseq_btree,
@@ -367,7 +372,6 @@ write_kvs(State, UpdateSeq, ViewKVs, DocIdKeys, Log) ->
         log_btree=LogBtree2
     }.
 
-
 update_id_btree(Btree, DocIdKeys, true) ->
     ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- DocIdKeys, DIKeys /= []],
     couch_btree:query_modify(Btree, [], ToAdd, []);
@@ -377,64 +381,59 @@ update_id_btree(Btree, DocIdKeys, _) ->
     ToRem = [Id || {Id, DIKeys} <- DocIdKeys, DIKeys == []],
     couch_btree:query_modify(Btree, ToFind, ToAdd, ToRem).
 
-update_log(Btree, Log, _UpdatedSeq, true) ->
-    ToAdd = lists:flatmap(fun({{Id, _Rev}, Keys}) ->
-        case Keys of
-            [] ->
-                [];
-            _ ->
-                [{Id, Keys}]
-        end
-    end, dict:to_list(Log)),
+
+update_log(Btree, Log, _Revs, _Seqs, true) ->
+    ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log),
+                             DIKeys /= []],
     {ok, LogBtree2} = couch_btree:add_remove(Btree, ToAdd, []),
     {ok, dict:new(), dict:new(), LogBtree2};
-update_log(Btree, Log, UpdatedSeq, _) ->
+update_log(Btree, Log, Revs, Seqs, _) ->
     %% build list of updated keys and Id
-    Revs = dict:from_list(dict:fetch_keys(Log)),
-    Log0 = dict:fold(fun({Id, _Rev}, DIKeys, Acc) ->
-        dict:store(Id, DIKeys, Acc)
-    end, dict:new(), Log),
-    {ToLook, Updated, Removed} = dict:fold(fun(Id, DIKeys, {IdsAcc, KeysAcc, RemAcc}) ->
-        {KeysAcc1, RemAcc1} = lists:foldl(fun({ViewId, {Key, _Seq, Op}}, Acc) ->
-            {KeysAcc2, RemAcc2} = Acc,
-            case Op of
-                add ->
-                    {[{Id, ViewId, Key} | KeysAcc2], RemAcc2};
-                del ->
-                    {KeysAcc2, [{Id, ViewId, Key} | RemAcc2]}
-            end
-        end, {KeysAcc, RemAcc}, DIKeys),
-        {[Id | IdsAcc], KeysAcc1, RemAcc1}
-    end, {[], [], []}, Log0),
+    {ToLook, Updated, Removed} = dict:fold(
+        fun(Id, [], {IdsAcc, KeysAcc, RemAcc}) ->
+            {[Id | IdsAcc], KeysAcc, RemAcc};
+        (Id, DIKeys, {IdsAcc, KeysAcc, RemAcc}) ->
+            {KeysAcc1, RemAcc1} = lists:foldl(fun({ViewId, {Key, _Seq, Op}}, {KeysAcc2, RemAcc2}) ->
+                case Op of
+                    add -> {[{Id, ViewId, Key}|KeysAcc2], RemAcc2};
+                    del -> {KeysAcc2, [{Id, ViewId, Key}|RemAcc2]}
+                end
+            end, {KeysAcc, RemAcc}, DIKeys),
+            {[Id | IdsAcc], KeysAcc1, RemAcc1}
+        end, {[], [], []}, Log),
 
     MapFun = fun({ok, KV}) -> [KV]; (not_found) -> [] end,
     KVsToLook = lists:flatmap(MapFun, couch_btree:lookup(Btree, ToLook)),
+
     {Log1, AddAcc, DelAcc} = lists:foldl(fun({DocId, VIdKeys}, Acc) ->
-        lists:foldl(fun({ViewId, {Key, Seq, _Op}}, {Log4, AddAcc4, DelAcc4}) ->
-            case lists:member({DocId, ViewId, Key}, Updated) of
+        lists:foldl(fun({ViewId, {Key, OldSeq, _Op}}, {Log4, AddAcc4, DelAcc4}) ->
+
+            IsUpdated = lists:member({DocId, ViewId, Key}, Updated),
+            IsRemoved = lists:member({DocId, ViewId, Key}, Removed),
+
+            case IsUpdated of
                 true ->
                     % the log is updated, deleted old record from the view
-                    DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
+                    DelAcc5 = dict:append(ViewId, {Key, OldSeq, DocId}, DelAcc4),
                     {Log4, AddAcc4, DelAcc5};
                 false ->
                     % an update operation has been logged for this key. We must
                     % now record it as deleted in the log, remove the old record
                     % in the view and update the view with a removed record.
-                    Log5 = case lists:member({DocId, ViewId, Key}, Removed) of
+                    NewSeq = dict:fetch(DocId, Seqs),
+                    Log5 = case IsRemoved of
                         false ->
-                            LogValue = {ViewId, {Key, UpdatedSeq, del}},
-                            dict:append(DocId, LogValue, Log4);
+                            dict:append(DocId, {ViewId, {Key, NewSeq, del}}, Log4);
                         true ->
                             Log4
                     end,
-                    DelAcc5 = dict:append(ViewId, {Key, Seq, DocId}, DelAcc4),
                     Rev = dict:fetch(DocId, Revs),
-                    AddValue = {{UpdatedSeq, Key}, {DocId, ?REM_VAL, Rev}},
-                    AddAcc5 = dict:append(ViewId, AddValue, AddAcc4),
+                    DelAcc5 = dict:append(ViewId, {Key, OldSeq, DocId}, DelAcc4),
+                    AddAcc5 = dict:append(ViewId, {{NewSeq, Key}, {DocId, ?REM_VAL, Rev}}, AddAcc4),
                     {Log5, AddAcc5, DelAcc5}
             end
         end, Acc, VIdKeys)
-    end, {Log0, dict:new(), dict:new()}, KVsToLook),
+    end, {Log, dict:new(), dict:new()}, KVsToLook),
 
     ToAdd = [{Id, DIKeys} || {Id, DIKeys} <- dict:to_list(Log1), DIKeys /= []],
     % store the new logs
diff --git a/src/couch_mrview_util.erl b/src/couch_mrview_util.erl
index 6054e3a..9669736 100644
--- a/src/couch_mrview_util.erl
+++ b/src/couch_mrview_util.erl
@@ -289,8 +289,7 @@ open_view(Db, Fd, Lang, {BTState, SeqBTState, KSeqBTState, USeq, PSeq}, View) ->
 
     BySeqReduceFun = fun couch_db_updater:btree_by_seq_reduce/2,
     {ok, SeqBtree} = if View#mrview.seq_indexed ->
-        ViewSeqBtOpts = [{less, fun less_json_seqs/2},
-                         {reduce, BySeqReduceFun},
+        ViewSeqBtOpts = [{reduce, BySeqReduceFun},
                          {compression, couch_db:compression(Db)}],
 
         couch_btree:open(SeqBTState, Fd, ViewSeqBtOpts);

From 4f925443fb3ffe2bcc82971cad1844f94af3ec4f Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Fri, 19 Sep 2014 03:52:15 -0700
Subject: [PATCH 39/41] remove unnecessary event type

---
 src/couch_mrview_index.erl | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/couch_mrview_index.erl b/src/couch_mrview_index.erl
index 893ecd1..7113bb1 100644
--- a/src/couch_mrview_index.erl
+++ b/src/couch_mrview_index.erl
@@ -186,9 +186,7 @@ finish_update(State) ->
 
 commit(State) ->
     Header = {State#mrst.sig, couch_mrview_util:make_header(State)},
-    Resp = couch_file:write_header(State#mrst.fd, Header),
-    couch_event:notify(State#mrst.db_name, {index_update, State#mrst.idx_name}),
-    Resp.
+    couch_file:write_header(State#mrst.fd, Header).
 
 
 compact(Db, State, Opts) ->

From e928480d81c16d0959b6c913123c750fb6efbb0f Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Tue, 21 Oct 2014 12:57:11 -0700
Subject: [PATCH 40/41] Change log macro to couch_log call

---
 src/couch_mrview_update_notifier.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/couch_mrview_update_notifier.erl b/src/couch_mrview_update_notifier.erl
index 1837b1f..7c7ff1b 100644
--- a/src/couch_mrview_update_notifier.erl
+++ b/src/couch_mrview_update_notifier.erl
@@ -42,7 +42,7 @@ handle_call(_Request, State) ->
     {reply, ok, State}.
 
 handle_info({'EXIT', Pid, Reason}, Pid) ->
-    ?LOG_ERROR("View update notification process ~p died: ~p", [Pid, Reason]),
+    couch_log:error("View update notification process ~p died: ~p", [Pid, Reason]),
     remove_handler.
 
 code_change(_OldVsn, State, _Extra) ->

From 28e51f3f8cfefce6300243ec6ce1af35d4f454ea Mon Sep 17 00:00:00 2001
From: Benjamin Bastian <benjamin.bastian@gmail.com>
Date: Thu, 30 Oct 2014 13:16:44 -0700
Subject: [PATCH 41/41] Update expected return of couch_mrview_util:get_view

---
 src/couch_mrview.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/couch_mrview.erl b/src/couch_mrview.erl
index f6dec66..26d1cb2 100644
--- a/src/couch_mrview.erl
+++ b/src/couch_mrview.erl
@@ -215,8 +215,8 @@ trigger_update(Db, DDoc, UpdateSeq) ->
 
 %% get informations on a view
 get_view_info(Db, DDoc, VName) ->
-    {ok, {_, View}, _, _Args} = couch_mrview_util:get_view(Db, DDoc, VName,
-                                                          #mrargs{}),
+    {ok, {_, View, _}, _, _Args} = couch_mrview_util:get_view(Db, DDoc, VName,
+                                                              #mrargs{}),
 
     %% get the total number of rows
     {ok, TotalRows} =  couch_mrview_util:get_row_count(View),