Permalink
Browse files

CBD-426 More efficient query path for new index format

Restore the performance/efficiency of the query path to at
least match the one we had prior to introducing an Erlang
termless index file format.

Change-Id: I168cfac36784c209117e07d2ccdbc27ef710bd12
Reviewed-on: http://review.couchbase.org/19212
Tested-by: Filipe David Borba Manana <fdmanana@gmail.com>
Reviewed-by: Filipe David Borba Manana <fdmanana@gmail.com>
  • Loading branch information...
1 parent e387087 commit 45b099bc0853ca553e0e20b7c4a860720423786b @fdmanana fdmanana committed with fdmanana Aug 2, 2012
View
12 src/couch_index_merger/src/couch_http_view_streamer.erl
@@ -76,36 +76,36 @@ next_streamer_state(Ctx, DataFun) ->
% _all_docs error row
transform_row({{Key, error}, Reason}, _Url) ->
RowJson = <<"{\"key\":", Key/binary, ",\"error\":", Reason/binary, "}">>,
- {{?JSON_DECODE(Key), error}, {row_json, RowJson}};
+ {{{json, Key}, error}, {row_json, RowJson}};
% map view rows
transform_row({{Key, DocId}, Value}, _Url) when is_binary(Value) ->
RowJson = <<"{\"id\":", DocId/binary, ",\"key\":", Key/binary,
",\"value\":", Value/binary, "}">>,
- {{?JSON_DECODE(Key), ?JSON_DECODE(DocId)}, {row_json, RowJson}};
+ {{{json, Key}, ?JSON_DECODE(DocId)}, {row_json, RowJson}};
transform_row({{Key, DocId}, Value, Doc}, _Url) when is_binary(Value) ->
RowJson = <<"{\"id\":", DocId/binary, ",\"key\":", Key/binary,
",\"value\":", Value/binary, ",\"doc\":", Doc/binary, "}">>,
- {{?JSON_DECODE(Key), ?JSON_DECODE(DocId)}, {row_json, RowJson}};
+ {{{json, Key}, ?JSON_DECODE(DocId)}, {row_json, RowJson}};
transform_row({{Key, DocId}, {PartId, _Node, Value}}, Url) ->
RowJson = <<"{\"id\":", DocId/binary, ",\"key\":", Key/binary,
",\"partition\":", PartId/binary, ",\"node\":", Url/binary,
",\"value\":", Value/binary, "}">>,
- {{?JSON_DECODE(Key), ?JSON_DECODE(DocId)}, {row_json, RowJson}};
+ {{{json, Key}, ?JSON_DECODE(DocId)}, {row_json, RowJson}};
transform_row({{Key, DocId}, {PartId, _Node, Value}, Doc}, Url) ->
RowJson = <<"{\"id\":", DocId/binary, ",\"key\":", Key/binary,
",\"partition\":", PartId/binary, ",\"node\":", Url/binary,
",\"value\":", Value/binary, ",\"doc\":", Doc/binary, "}">>,
- {{?JSON_DECODE(Key), ?JSON_DECODE(DocId)}, {row_json, RowJson}};
+ {{{json, Key}, ?JSON_DECODE(DocId)}, {row_json, RowJson}};
% reduce view rows
transform_row({Key, Value}, _Url) when is_binary(Key) ->
RowJson = <<"{\"key\":", Key/binary, ",\"value\":", Value/binary, "}">>,
% value_json, in case rereduce needs to be done by the merger
- {?JSON_DECODE(Key), {row_json, RowJson}, {value_json, Value}}.
+ {{json, Key}, {row_json, RowJson}, {value_json, Value}}.
make_error_item({_Node, Reason}, Url) ->
View
110 src/couch_index_merger/src/couch_view_merger.erl
@@ -38,7 +38,7 @@
% callback!
parse_http_params(Req, DDoc, ViewName, #view_merge{keys = Keys}) ->
% view type =~ query type
- {_Collation, ViewType0} = view_details(DDoc, ViewName),
+ ViewType0 = view_type(DDoc, ViewName),
ViewType = case {ViewType0, couch_httpd:qs_value(Req, "reduce", "true")} of
{reduce, "false"} ->
red_map;
@@ -61,18 +61,23 @@ parse_http_params(Req, DDoc, ViewName, #view_merge{keys = Keys}) ->
make_funs(DDoc, ViewName, IndexMergeParams) ->
#index_merge{
extra = Extra,
- http_params = #view_query_args{debug = DebugMode} = ViewArgs
+ http_params = ViewArgs
} = IndexMergeParams,
#view_merge{
rereduce_fun = InRedFun,
make_row_fun = MakeRowFun0
} = Extra,
- {Collation, ViewType0} = view_details(DDoc, ViewName),
- ViewType = case {ViewType0, ViewArgs#view_query_args.run_reduce} of
- {reduce, false} ->
- red_map;
- _ ->
- ViewType0
+ #view_query_args{
+ debug = DebugMode,
+ view_type = ViewType0,
+ direction = Dir
+ } = ViewArgs,
+ Collation = view_collation(ViewName),
+ ViewType = case ViewType0 of
+ nil ->
+ view_type(DDoc, ViewName);
+ _ when ViewType0 == map; ViewType0 == red_map; ViewType0 == reduce ->
+ ViewType0
end,
RedFun = case {ViewType, InRedFun} of
{reduce, nil} ->
@@ -82,8 +87,7 @@ make_funs(DDoc, ViewName, IndexMergeParams) ->
_ ->
nil
end,
- LessFun = view_less_fun(Collation, ViewArgs#view_query_args.direction,
- ViewType),
+ LessFun = view_less_fun(Collation, Dir, ViewType, view_class(IndexMergeParams)),
{FoldFun, MergeFun} = case ViewType of
reduce ->
{fun reduce_view_folder/6, fun merge_reduce_views/1};
@@ -211,20 +215,31 @@ http_index_folder_req_details(#simple_index_spec{} = IndexSpec, MergeParams, _DD
end.
-view_details(nil, <<"_all_docs">>) ->
- {<<"raw">>, map};
-
-view_details(DDoc, ViewName) ->
+view_type(nil, <<"_all_docs">>) ->
+ map;
+view_type(DDoc, ViewName) ->
{ViewDef} = get_view_def(DDoc, ViewName),
- {ViewOptions} = get_value(<<"options">>, ViewDef, {[]}),
- Collation = get_value(<<"collation">>, ViewOptions, <<"default">>),
- ViewType = case get_value(<<"reduce">>, ViewDef) of
+ case get_value(<<"reduce">>, ViewDef) of
undefined ->
map;
RedFun when is_binary(RedFun) ->
reduce
- end,
- {Collation, ViewType}.
+ end.
+
+view_collation(<<"_all_docs">>) ->
+ raw;
+view_collation(_ViewName) ->
+ default.
+
+
+view_class(#index_merge{indexes = Specs}) ->
+ view_class(Specs);
+view_class([#set_view_spec{} | _]) ->
+ set_view;
+view_class([#simple_index_spec{} | _]) ->
+ normal;
+view_class([#merged_index_spec{} | Rest]) ->
+ view_class(Rest).
reduce_function(#doc{id = DDocId} = DDoc, ViewName) ->
@@ -250,27 +265,30 @@ get_view_def(#doc{body = DDoc, id = DDocId}, ViewName) ->
end.
-view_less_fun(Collation, Dir, ViewType) ->
- LessFun = case Collation of
- <<"default">> ->
- case ViewType of
- _ when ViewType =:= map; ViewType =:= red_map ->
- fun(RowA, RowB) ->
- couch_view:less_json_ids(element(1, RowA), element(1, RowB))
- end;
- reduce ->
- fun(RowA, RowB) ->
- couch_view:less_json(element(1, RowA), element(1, RowB))
- end
+view_less_fun(Collation, Dir, ViewType, ViewClass) ->
+ LessFun = case ViewClass of
+ normal when ViewType == reduce, Collation == default ->
+ fun couch_view:less_json/2;
+ normal when (ViewType == map orelse ViewType == red_map), Collation == default ->
+ fun couch_view:less_json_ids/2;
+ normal when ViewType == map, Collation == raw ->
+ % _all_docs
+ fun({_Key1, Id1}, {_Key2, Id2}) when is_binary(Id1), is_binary(Id2) ->
+ Id1 < Id2;
+ ({{json, Key1}, _Id1}, {{json, Key2}, _Id2}) ->
+ % Id1/Id2 is 'error' (POST requests to _all_docs with keys parameter)
+ ?JSON_DECODE(Key1) < ?JSON_DECODE(Key2)
end;
- <<"raw">> ->
- fun(A, B) -> element(1, A) < element(1, B) end
+ set_view when ViewType == reduce, Collation == default ->
+ fun couch_set_view:reduce_view_key_compare/2;
+ set_view when (ViewType == map orelse ViewType == red_map), Collation == default ->
+ fun couch_set_view:map_view_key_compare/2
end,
case Dir of
fwd ->
- LessFun;
+ fun(RowA, RowB) -> LessFun(element(1, RowA), element(1, RowB)) end;
rev ->
- fun(A, B) -> not LessFun(A, B) end
+ fun(RowA, RowB) -> not LessFun(element(1, RowA), element(1, RowB)) end
end.
% Optimized path, row assembled by couch_http_view_streamer
@@ -784,8 +802,7 @@ fold_local_all_docs(nil, Db, Queue, ViewArgs) ->
end_docid = EndDocId,
direction = Dir,
inclusive_end = InclusiveEnd,
- include_docs = IncludeDocs,
- conflicts = Conflicts
+ include_docs = IncludeDocs
} = ViewArgs,
StartId = if is_binary(StartKey) -> StartKey;
true -> StartDocId
@@ -802,7 +819,7 @@ fold_local_all_docs(nil, Db, Queue, ViewArgs) ->
#doc_info{deleted = true} ->
ok;
_ ->
- Row = all_docs_row(DocInfo, Db, IncludeDocs, Conflicts),
+ Row = all_docs_row(DocInfo, Db, IncludeDocs),
ok = couch_view_merger_queue:queue(Queue, Row)
end,
{ok, Acc}
@@ -812,8 +829,7 @@ fold_local_all_docs(nil, Db, Queue, ViewArgs) ->
fold_local_all_docs(Keys, Db, Queue, ViewArgs) ->
#view_query_args{
direction = Dir,
- include_docs = IncludeDocs,
- conflicts = Conflicts
+ include_docs = IncludeDocs
} = ViewArgs,
FoldFun = case Dir of
fwd ->
@@ -825,34 +841,34 @@ fold_local_all_docs(Keys, Db, Queue, ViewArgs) ->
fun(Key, _Acc) ->
Row = case (catch couch_db:get_doc_info(Db, Key)) of
{ok, #doc_info{} = DocInfo} ->
- all_docs_row(DocInfo, Db, IncludeDocs, Conflicts);
+ all_docs_row(DocInfo, Db, IncludeDocs);
not_found ->
- {{Key, error}, not_found}
+ {{{json, ?JSON_ENCODE(Key)}, error}, not_found}
end,
ok = couch_view_merger_queue:queue(Queue, Row)
end, [], Keys).
-all_docs_row(DocInfo, Db, IncludeDoc, Conflicts) ->
+all_docs_row(DocInfo, Db, IncludeDoc) ->
#doc_info{id = Id, rev = Rev, deleted = Del} = DocInfo,
Value = {[{<<"rev">>, couch_doc:rev_to_str(Rev)}] ++ case Del of
true ->
[{<<"deleted">>, true}];
false ->
[]
end},
+ Key = {{json, ?JSON_ENCODE(Id)}, Id},
case IncludeDoc of
true ->
case Del of
true ->
DocVal = null;
false ->
- DocOptions = if Conflicts -> [conflicts]; true -> [] end,
- [{doc, DocVal}] = couch_httpd_view:doc_member(Db, DocInfo, DocOptions)
+ [{doc, DocVal}] = couch_httpd_view:doc_member(Db, DocInfo, [])
end,
- {{Id, Id}, Value, DocVal};
+ {Key, Value, DocVal};
false ->
- {{Id, Id}, Value}
+ {Key, Value}
end.
View
60 src/couch_set_view/src/couch_set_view.erl
@@ -29,7 +29,7 @@
-export([fold/5, fold_reduce/5]).
-export([get_row_count/2, reduce_to_count/1, extract_map_view/1]).
--export([less_json/2, less_json_ids/2]).
+-export([map_view_key_compare/2, reduce_view_key_compare/2]).
-export([handle_db_event/1]).
@@ -532,23 +532,23 @@ do_fold_reduce(Group, ViewInfo, Fun, Acc, Options0, ViewQueryArgs) ->
WrapperFun = fun(KeyDocId, PartialReds, Acc0) ->
GroupedKey = case GroupLevel of
0 ->
- null;
+ <<"null">>;
_ when is_integer(GroupLevel) ->
- {Key, _DocId} = couch_set_view_util:decode_key_docid(KeyDocId),
- case is_list(Key) of
+ {KeyJson, _DocId} = couch_set_view_util:split_key_docid(KeyDocId),
+ case is_array_key(KeyJson) of
true ->
- lists:sublist(Key, GroupLevel);
+ ?JSON_ENCODE(lists:sublist(?JSON_DECODE(KeyJson), GroupLevel));
false ->
- Key
+ KeyJson
end;
_ ->
- {Key, _DocId} = couch_set_view_util:decode_key_docid(KeyDocId),
- Key
+ {KeyJson, _DocId} = couch_set_view_util:split_key_docid(KeyDocId),
+ KeyJson
end,
<<_Count:40, _BitMap:?MAX_NUM_PARTITIONS, Reds/binary>> =
couch_btree:final_reduce(ReduceFun, PartialReds),
UserRed = lists:nth(NthRed, couch_set_view_util:parse_reductions(Reds)),
- Fun(GroupedKey, {json, UserRed}, Acc0)
+ Fun({json, GroupedKey}, {json, UserRed}, Acc0)
end,
couch_set_view_util:open_raw_read_fd(Group),
try
@@ -714,8 +714,8 @@ fold_fun(_Fun, [], _, Acc) ->
{ok, Acc};
fold_fun(Fun, [KV | Rest], {KVReds, Reds}, Acc) ->
{KeyDocId, <<PartId:16, Value/binary>>} = KV,
- {Key, DocId} = couch_set_view_util:decode_key_docid(KeyDocId),
- case Fun({{Key, DocId}, {PartId, {json, Value}}}, {KVReds, Reds}, Acc) of
+ {JsonKey, DocId} = couch_set_view_util:split_key_docid(KeyDocId),
+ case Fun({{{json, JsonKey}, DocId}, {PartId, {json, Value}}}, {KVReds, Reds}, Acc) of
{ok, Acc2} ->
fold_fun(Fun, Rest, {[KV | KVReds], Reds}, Acc2);
{stop, Acc2} ->
@@ -901,17 +901,17 @@ nuke_dir(RootDelDir, Dir) ->
end.
-% keys come back in the language of btree - tuples.
-less_json_ids({JsonA, IdA}, {JsonB, IdB}) ->
- case couch_ejson_compare:less(JsonA, JsonB) of
+map_view_key_compare({Key1, DocId1}, {Key2, DocId2}) ->
+ case couch_ejson_compare:less_json(Key1, Key2) of
0 ->
- IdA < IdB;
- Result ->
- Result < 0
+ DocId1 < DocId2;
+ LessResult ->
+ LessResult < 0
end.
-less_json(A,B) ->
- couch_ejson_compare:less(A, B) < 0.
+
+reduce_view_key_compare(A, B) ->
+ couch_ejson_compare:less_json(A, B) < 0.
modify_bitmasks(Group, []) ->
@@ -1065,18 +1065,24 @@ make_reduce_group_keys_fun(0) ->
fun(_, _) -> true end;
make_reduce_group_keys_fun(GroupLevel) when is_integer(GroupLevel) ->
fun(KeyDocId1, KeyDocId2) ->
- {Key1, _DocId1} = couch_set_view_util:decode_key_docid(KeyDocId1),
- {Key2, _DocId2} = couch_set_view_util:decode_key_docid(KeyDocId2),
- case is_list(Key1) andalso is_list(Key2) of
+ {Key1, _DocId1} = couch_set_view_util:split_key_docid(KeyDocId1),
+ {Key2, _DocId2} = couch_set_view_util:split_key_docid(KeyDocId2),
+ case is_array_key(Key1) andalso is_array_key(Key2) of
true ->
- lists:sublist(Key1, GroupLevel) == lists:sublist(Key2, GroupLevel);
+ lists:sublist(?JSON_DECODE(Key1), GroupLevel) ==
+ lists:sublist(?JSON_DECODE(Key2), GroupLevel);
false ->
- Key1 == Key2
+ couch_ejson_compare:less_json(Key1, Key2) == 0
end
end;
make_reduce_group_keys_fun(_) ->
fun(KeyDocId1, KeyDocId2) ->
- {Key1, _DocId1} = couch_set_view_util:decode_key_docid(KeyDocId1),
- {Key2, _DocId2} = couch_set_view_util:decode_key_docid(KeyDocId2),
- Key1 == Key2
+ {Key1, _DocId1} = couch_set_view_util:split_key_docid(KeyDocId1),
+ {Key2, _DocId2} = couch_set_view_util:split_key_docid(KeyDocId2),
+ couch_ejson_compare:less_json(Key1, Key2) == 0
end.
+
+is_array_key(<<"[", _/binary>>) ->
+ true;
+is_array_key(K) when is_binary(K) ->
+ false.
View
2 src/couch_set_view/test/15-passive-partitions.t
@@ -347,7 +347,7 @@ fold_view(ActiveParts, ValueGenFun) ->
etap:diag("Folding the view with ?group=true"),
PerKeyRedFoldFun = fun(Key, {json, Red}, {NextVal, I}) ->
- ExpectedKey = doc_id(NextVal),
+ ExpectedKey = {json, ejson:encode(doc_id(NextVal))},
ExpectedRed = ValueGenFun(NextVal),
case {Key, ejson:decode(Red)} of
{ExpectedKey, ExpectedRed} ->
View
13 src/couchdb/couch_ejson_compare.erl
@@ -16,6 +16,8 @@
-on_load(init/0).
+-type raw_json() :: binary() | iolist() | {'json', binary()} | {'json', iolist()}.
+
init() ->
LibDir = case couch_config:get("couchdb", "util_driver_dir") of
@@ -47,10 +49,9 @@ less(A, B) ->
end.
--spec less_json(RawJsonKey1::iolist() | binary(),
- RawJsonKey2::iolist() | binary()) -> -1 .. 1.
+-spec less_json(raw_json(), raw_json()) -> -1 .. 1.
less_json(A, B) ->
- case less_json_nif(A, B) of
+ case less_json_nif(get_raw_json(A), get_raw_json(B)) of
{error, _Reason} = Error ->
throw(Error);
Else ->
@@ -124,3 +125,9 @@ less_list([A|RestA], [B|RestB]) ->
convert(N) when N < 0 -> -1;
convert(N) when N > 0 -> 1;
convert(_) -> 0.
+
+
+get_raw_json({json, RawJson}) ->
+ RawJson;
+get_raw_json(RawJson) ->
+ RawJson.

0 comments on commit 45b099b

Please sign in to comment.