Permalink
Browse files

Fixes COUCHDB-799 - More granular ETags for views.

ETags for views now only change when their underlying view index
changes due to indexing or purges. ETags are also specific to each
view.

Thanks to Klaus Trainer for the patch.



git-svn-id: https://svn.apache.org/repos/asf/couchdb/trunk@1006339 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
1 parent 152f2b9 commit c67ba1f96daaec66b01c80b27ff92f33696b0900 @davisp davisp committed Oct 10, 2010
View
1 THANKS
@@ -68,5 +68,6 @@ suggesting improvements or submitting changes. Some of these people are:
* David Rose <doppler@gmail.com>
* Lim Yue Chuan <shasderias@gmail.com>
* David Davis <xantus@xantus.org>
+ * Klaus Trainer <klaus.trainer@web.de>
For a list of authors see the `AUTHORS` file.
@@ -110,7 +110,7 @@ function() {
var vinfo = dinfo.view_index;
TEquals(51, vinfo.disk_size);
TEquals(false, vinfo.compact_running);
- TEquals("dc3264b45b74cc6d94666e3043e07154", vinfo.signature, 'ddoc sig');
+ TEquals("86e9b34892b4df35cd2f7c27da30c94d", vinfo.signature, 'ddoc sig');
db.bulkSave(makeDocs(1, numDocs + 1));
@@ -11,23 +11,34 @@
// the License.
couchTests.etags_views = function(debug) {
- var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"false"});
+ var db = new CouchDB("test_suite_db", {"X-Couch-Full-Commit":"true"});
db.deleteDb();
db.createDb();
if (debug) debugger;
var designDoc = {
- _id:"_design/etags",
+ _id: "_design/etags",
language: "javascript",
views : {
+ fooView: {
+ map: stringFun(function(doc) {
+ if (doc.foo) {
+ emit("bar", 1);
+ }
+ }),
+ },
basicView : {
map : stringFun(function(doc) {
- emit(doc.integer, doc.string);
+ if(doc.integer && doc.string) {
+ emit(doc.integer, doc.string);
+ }
})
},
withReduce : {
map : stringFun(function(doc) {
- emit(doc.integer, doc.string);
+ if(doc.integer && doc.string) {
+ emit(doc.integer, doc.string);
+ }
}),
reduce : stringFun(function(keys, values, rereduce) {
if (rereduce) {
@@ -40,9 +51,9 @@ couchTests.etags_views = function(debug) {
}
};
T(db.save(designDoc).ok);
+ db.bulkSave(makeDocs(0, 10));
+
var xhr;
- var docs = makeDocs(0, 10);
- db.bulkSave(docs);
// verify get w/Etag on map view
xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
@@ -52,17 +63,92 @@ couchTests.etags_views = function(debug) {
headers: {"if-none-match": etag}
});
T(xhr.status == 304);
- // TODO GET with keys (when that is available)
+
+ // verify ETag doesn't change when an update
+ // doesn't change the view group's index
+ T(db.save({"_id":"doc1", "foo":"bar"}).ok);
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 == etag);
+
+ // Verify that purges affect etags
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+ var foo_etag = xhr.getResponseHeader("etag");
+ var doc1 = db.open("doc1");
+ xhr = CouchDB.request("POST", "/test_suite_db/_purge", {
+ body: JSON.stringify({"doc1":[doc1._rev]})
+ });
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 != foo_etag);
+
+ // Test that _purge didn't affect the other view etags.
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 == etag);
+
+ // verify different views in the same view group may have different ETags
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+ var etag1 = xhr.getResponseHeader("etag");
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag2 = xhr.getResponseHeader("etag");
+ T(etag1 != etag2);
+
+ // verify ETag changes when an update changes the view group's index.
+ db.bulkSave(makeDocs(10, 20));
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 != etag);
+
+ // verify ETag is the same after a restart
+ restartServer();
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/basicView");
+ var etag2 = xhr.getResponseHeader("etag");
+ T(etag1 == etag2);
// reduce view
xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
T(xhr.status == 200);
var etag = xhr.getResponseHeader("etag");
- xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce", {
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce",{
headers: {"if-none-match": etag}
});
T(xhr.status == 304);
+ // verify ETag doesn't change when an update
+ // doesn't change the view group's index
+ T(db.save({"_id":"doc3", "foo":"bar"}).ok);
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 == etag);
+ // purge
+ var doc3 = db.open("doc3");
+ xhr = CouchDB.request("POST", "/test_suite_db/_purge", {
+ body: JSON.stringify({"doc3":[doc3._rev]})
+ });
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 == etag);
+
+ // verify different views in the same view group may have different ETags
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/fooView");
+ var etag1 = xhr.getResponseHeader("etag");
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag2 = xhr.getResponseHeader("etag");
+ T(etag1 != etag2);
+
+ // verify ETag changes when an update changes the view group's index
+ db.bulkSave(makeDocs(20, 30));
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag1 = xhr.getResponseHeader("etag");
+ T(etag1 != etag);
+
+ // verify ETag is the same after a restart
+ restartServer();
+ xhr = CouchDB.request("GET", "/test_suite_db/_design/etags/_view/withReduce");
+ var etag2 = xhr.getResponseHeader("etag");
+ T(etag1 == etag2);
+
// confirm ETag changes with different POST bodies
xhr = CouchDB.request("POST", "/test_suite_db/_design/etags/_view/basicView",
{body: JSON.stringify({keys:[1]})}
View
@@ -238,6 +238,8 @@
-record(view,
{id_num,
+ update_seq=0,
+ purge_seq=0,
map_names=[],
def,
btree=nil,
@@ -188,14 +188,14 @@ handle_view_list_req(Req, _Db, _DDoc) ->
handle_view_list(Req, Db, DDoc, LName, {ViewDesignName, ViewName}, Keys) ->
ViewDesignId = <<"_design/", ViewDesignName/binary>>,
{ViewType, View, Group, QueryArgs} = couch_httpd_view:load_view(Req, Db, {ViewDesignId, ViewName}, Keys),
- Etag = list_etag(Req, Db, Group, {couch_httpd:doc_etag(DDoc), Keys}),
+ Etag = list_etag(Req, Db, Group, View, {couch_httpd:doc_etag(DDoc), Keys}),
couch_httpd:etag_respond(Req, Etag, fun() ->
output_list(ViewType, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group)
end).
-list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, More) ->
+list_etag(#httpd{user_ctx=UserCtx}=Req, Db, Group, View, More) ->
Accept = couch_httpd:header_value(Req, "Accept"),
- couch_httpd_view:view_group_etag(Group, Db, {More, Accept, UserCtx#user_ctx.roles}).
+ couch_httpd_view:view_etag(Db, Group, View, {More, Accept, UserCtx#user_ctx.roles}).
output_list(map, Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group) ->
output_map_list(Req, Db, DDoc, LName, View, QueryArgs, Etag, Keys, Group);
@@ -17,7 +17,7 @@
-export([parse_view_params/3]).
-export([make_view_fold_fun/7, finish_view_fold/4, finish_view_fold/5, view_row_obj/3]).
--export([view_group_etag/2, view_group_etag/3, make_reduce_fold_funs/6]).
+-export([view_etag/3, view_etag/4, make_reduce_fold_funs/6]).
-export([design_doc_view/5, parse_bool_param/1, doc_member/2]).
-export([make_key_options/1, load_view/4]).
@@ -113,7 +113,7 @@ output_map_view(Req, View, Group, Db, QueryArgs, nil) ->
limit = Limit,
skip = SkipCount
} = QueryArgs,
- CurrentEtag = view_group_etag(Group, Db),
+ CurrentEtag = view_etag(Db, Group, View),
couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
{ok, RowCount} = couch_view:get_row_count(View),
FoldlFun = make_view_fold_fun(Req, QueryArgs, CurrentEtag, Db, Group#group.current_seq, RowCount, #view_fold_helper_funs{reduce_count=fun couch_view:reduce_to_count/1}),
@@ -129,7 +129,7 @@ output_map_view(Req, View, Group, Db, QueryArgs, Keys) ->
limit = Limit,
skip = SkipCount
} = QueryArgs,
- CurrentEtag = view_group_etag(Group, Db, Keys),
+ CurrentEtag = view_etag(Db, Group, View, Keys),
couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
{ok, RowCount} = couch_view:get_row_count(View),
FoldAccInit = {Limit, SkipCount, undefined, []},
@@ -154,7 +154,7 @@ output_reduce_view(Req, Db, View, Group, QueryArgs, nil) ->
skip = Skip,
group_level = GroupLevel
} = QueryArgs,
- CurrentEtag = view_group_etag(Group, Db),
+ CurrentEtag = view_etag(Db, Group, View),
couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
{ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel,
QueryArgs, CurrentEtag, Group#group.current_seq,
@@ -172,7 +172,7 @@ output_reduce_view(Req, Db, View, Group, QueryArgs, Keys) ->
skip = Skip,
group_level = GroupLevel
} = QueryArgs,
- CurrentEtag = view_group_etag(Group, Db, Keys),
+ CurrentEtag = view_etag(Db, Group, View, Keys),
couch_httpd:etag_respond(Req, CurrentEtag, fun() ->
{ok, GroupRowsFun, RespFun} = make_reduce_fold_funs(Req, GroupLevel,
QueryArgs, CurrentEtag, Group#group.current_seq,
@@ -607,16 +607,15 @@ send_json_reduce_row(Resp, {Key, Value}, RowFront) ->
send_chunk(Resp, RowFront ++ ?JSON_ENCODE({[{key, Key}, {value, Value}]})),
{ok, ",\r\n"}.
-view_group_etag(Group, Db) ->
- view_group_etag(Group, Db, nil).
+view_etag(Db, Group, View) ->
+ view_etag(Db, Group, View, nil).
-view_group_etag(#group{sig=Sig,current_seq=CurrentSeq}, _Db, Extra) ->
- % ?LOG_ERROR("Group ~p",[Group]),
- % This is not as granular as it could be.
- % If there are updates to the db that do not effect the view index,
- % they will change the Etag. For more granular Etags we'd need to keep
- % track of the last Db seq that caused an index change.
- couch_httpd:make_etag({Sig, CurrentSeq, Extra}).
+view_etag(Db, Group, {reduce, _, _, View}, Extra) ->
+ view_etag(Db, Group, View, Extra);
+view_etag(Db, Group, {temp_reduce, View}, Extra) ->
+ view_etag(Db, Group, View, Extra);
+view_etag(_Db, #group{sig=Sig}, #view{update_seq=UpdateSeq, purge_seq=PurgeSeq}, Extra) ->
+ couch_httpd:make_etag({Sig, UpdateSeq, PurgeSeq, Extra}).
% the view row has an error
view_row_obj(_Db, {{Key, error}, Value}, _IncludeDocs) ->
@@ -404,11 +404,15 @@ prepare_group({RootDir, DbName, #group{sig=Sig}=Group}, ForceReset)->
get_index_header_data(#group{current_seq=Seq, purge_seq=PurgeSeq,
id_btree=IdBtree,views=Views}) ->
- ViewStates = [couch_btree:get_state(Btree) || #view{btree=Btree} <- Views],
- #index_header{seq=Seq,
- purge_seq=PurgeSeq,
- id_btree_state=couch_btree:get_state(IdBtree),
- view_states=ViewStates}.
+ ViewStates = [
+ {couch_btree:get_state(V#view.btree), V#view.update_seq, V#view.purge_seq} || V <- Views
+ ],
+ #index_header{
+ seq=Seq,
+ purge_seq=PurgeSeq,
+ id_btree_state=couch_btree:get_state(IdBtree),
+ view_states=ViewStates
+ }.
hex_sig(GroupSig) ->
couch_util:to_hex(?b2l(GroupSig)).
@@ -460,13 +464,15 @@ set_view_sig(#group{
lib={[]},
def_lang=Language,
design_options=DesignOptions}=G) ->
- G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions}))};
+ ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views],
+ G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions}))};
set_view_sig(#group{
views=Views,
lib=Lib,
def_lang=Language,
design_options=DesignOptions}=G) ->
- G#group{sig=couch_util:md5(term_to_binary({Views, Language, DesignOptions, sort_lib(Lib)}))}.
+ ViewInfo = [V#view{update_seq=0, purge_seq=0} || V <- Views],
+ G#group{sig=couch_util:md5(term_to_binary({ViewInfo, Language, DesignOptions, sort_lib(Lib)}))}.
sort_lib({Lib}) ->
sort_lib(Lib, []).
@@ -574,14 +580,14 @@ delete_index_file(RootDir, DbName, GroupSig) ->
init_group(Db, Fd, #group{views=Views}=Group, nil) ->
init_group(Db, Fd, Group,
#index_header{seq=0, purge_seq=couch_db:get_purge_seq(Db),
- id_btree_state=nil, view_states=[nil || _ <- Views]});
+ id_btree_state=nil, view_states=[{nil, 0, 0} || _ <- Views]});
init_group(Db, Fd, #group{def_lang=Lang,views=Views}=
Group, IndexHeader) ->
#index_header{seq=Seq, purge_seq=PurgeSeq,
id_btree_state=IdBtreeState, view_states=ViewStates} = IndexHeader,
{ok, IdBtree} = couch_btree:open(IdBtreeState, Fd),
Views2 = lists:zipwith(
- fun(BtreeState, #view{reduce_funs=RedFuns,options=Options}=View) ->
+ fun({BTState, USeq, PSeq}, #view{reduce_funs=RedFuns,options=Options}=View) ->
FunSrcs = [FunSrc || {_Name, FunSrc} <- RedFuns],
ReduceFun =
fun(reduce, KVs) ->
@@ -604,10 +610,10 @@ init_group(Db, Fd, #group{def_lang=Lang,views=Views}=
<<"raw">> ->
Less = fun(A,B) -> A < B end
end,
- {ok, Btree} = couch_btree:open(BtreeState, Fd,
- [{less, Less},
- {reduce, ReduceFun}]),
- View#view{btree=Btree}
+ {ok, Btree} = couch_btree:open(BTState, Fd,
+ [{less, Less}, {reduce, ReduceFun}]
+ ),
+ View#view{btree=Btree, update_seq=USeq, purge_seq=PSeq}
end,
ViewStates, Views),
Group#group{db=Db, fd=Fd, current_seq=Seq, purge_seq=PurgeSeq,
@@ -96,19 +96,25 @@ purge_index(#group{db=Db, views=Views, id_btree=IdBtree}=Group) ->
end, dict:new(), Lookups),
% Now remove the values from the btrees
+ PurgeSeq = couch_db:get_purge_seq(Db),
Views2 = lists:map(
fun(#view{id_num=Num,btree=Btree}=View) ->
case dict:find(Num, ViewKeysToRemoveDict) of
{ok, RemoveKeys} ->
- {ok, Btree2} = couch_btree:add_remove(Btree, [], RemoveKeys),
- View#view{btree=Btree2};
+ {ok, ViewBtree2} = couch_btree:add_remove(Btree, [], RemoveKeys),
+ case ViewBtree2 =/= Btree of
+ true ->
+ View#view{btree=ViewBtree2, purge_seq=PurgeSeq};
+ _ ->
+ View#view{btree=ViewBtree2}
+ end;
error -> % no keys to remove in this view
View
end
end, Views),
Group#group{id_btree=IdBtree2,
views=Views2,
- purge_seq=couch_db:get_purge_seq(Db)}.
+ purge_seq=PurgeSeq}.
load_doc(Db, DocInfo, MapQueue, DocOpts, IncludeDesign) ->
@@ -247,7 +253,12 @@ write_changes(Group, ViewKeyValuesToAdd, DocIdViewIdKeys, NewSeq, InitialBuild)
Views2 = lists:zipwith(fun(View, {_View, AddKeyValues}) ->
KeysToRemove = couch_util:dict_find(View#view.id_num, KeysToRemoveByView, []),
{ok, ViewBtree2} = couch_btree:add_remove(View#view.btree, AddKeyValues, KeysToRemove),
- View#view{btree = ViewBtree2}
+ case ViewBtree2 =/= View#view.btree of
+ true ->
+ View#view{btree=ViewBtree2, update_seq=NewSeq};
+ _ ->
+ View#view{btree=ViewBtree2}
+ end
end, Group#group.views, ViewKeyValuesToAdd),
Group#group{views=Views2, current_seq=NewSeq, id_btree=IdBtree2}.

0 comments on commit c67ba1f

Please sign in to comment.