Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

937 track conflicts in db info #78

Open
wants to merge 2 commits into from

4 participants

@bdionne

This is code we had before, ported to sync with the latest on master. We're now storing a 4-tuple in the by_id btree, it's been suggested by @davisp that we move to proplists/records.

Bob Dionne added some commits
@kocolosk
Owner

I agree that these tuples get unwieldy, but I'd rather not add too many bytes to the on-disk representation of the reductions. I wonder if it's possible to add some assemble/extract logic to the reductions similar to what we have for the KVs themselves?

@davisp
Owner

Also, the suggestion is for proplists on disk and records in memory. Writing records to disk is nearly as bad as writing bare tuples.

@Adam, there might be some sort of logic there, but the underlying issue that concerns me is when we get into trying to figure out compatibility/upgrade matrices and things start getting out of control. I see your point on extra bytes in reductions though. I was thinking about some sort of helper module that would allow for a translation. Perhaps we can have a thing that makes it easier for versioning these things. I'll try and keep that in the back of my mind for a couple hours.

@kocolosk
Owner

Cool. A versioned record would certainly be cheaper than a proplist on disk.

@adam

@davisp, love ya...but I'm not the adam you're looking for.

@davisp
Owner

@adam Oopsie, apologies for the noise. Just switched over from JIRA where we use @-notation on real names instead of user names.

@bdionne

@davisp I'm not sure what you mean by compatibility matrices, but I suspect what happens porting this to couchdb[1] is an example of that.

[1] bdionne/couchdb@48d8eab

@davisp
Owner

@bdionne More or less. I'm just saying that every time we add something like this, its a new entry on a compatibility axis that we have to check for. And then between BC and LC we have to do even more checking. It makes me sad.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Oct 18, 2011
  1. Fix whitespace

    Bob Dionne authored
Commits on Oct 20, 2011
  1. Compute and expose document conflict count in db_info api

    Bob Dionne authored
    BugzID:937
This page is out of date. Refresh to see the latest.
View
41 apps/couch/src/couch_db.erl
@@ -154,7 +154,7 @@ apply_open_options({ok, Doc},Options) ->
apply_open_options2(Doc,Options);
apply_open_options(Else,_Options) ->
Else.
-
+
apply_open_options2(Doc,[]) ->
{ok, Doc};
apply_open_options2(#doc{atts=Atts,revs=Revs}=Doc,
@@ -256,8 +256,12 @@ get_last_purged(#db{fd=Fd, header=#db_header{purged_docs=PurgedPointer}}) ->
couch_file:pread_term(Fd, PurgedPointer).
get_doc_count(Db) ->
- {ok, {Count, _, _}} = couch_btree:full_reduce(Db#db.id_tree),
- {ok, Count}.
+ case couch_btree:full_reduce(Db#db.id_tree) of
+ {ok, {OCount, _, _}} ->
+ {ok, OCount};
+ {ok, {NCount, _, _, _}} ->
+ {ok, NCount}
+ end.
get_db_info(Db) ->
#db{fd=Fd,
@@ -269,11 +273,16 @@ get_db_info(Db) ->
instance_start_time=StartTime,
committed_update_seq=CommittedUpdateSeq} = Db,
{ok, Size} = couch_file:bytes(Fd),
- {ok, {Count, DelCount, DataSize}} = couch_btree:full_reduce(FullDocBtree),
+ {ok, {Count, DelCount, Conflicts, DataSize}} =
+ case couch_btree:full_reduce(FullDocBtree) of
+ {ok, {_,_,_,_}} = New -> New;
+ {ok, {C,De,Da}} -> {ok, {C,De,0,Da}}
+ end,
InfoList = [
{db_name, Name},
{doc_count, Count},
{doc_del_count, DelCount},
+ {conflicts_count, Conflicts},
{update_seq, SeqNum},
{purge_seq, couch_db:get_purge_seq(Db)},
{compact_running, Compactor/=nil},
@@ -618,11 +627,11 @@ prep_and_validate_replicated_updates(Db, [Bucket|RestBuckets], [OldInfo|RestOldI
{ok, {Start, Path}} ->
% our unflushed doc is a leaf node. Go back on the path
% to find the previous rev that's on disk.
-
+
LoadPrevRevFun = fun() ->
make_first_doc_on_disk(Db,Id,Start-1, tl(Path))
end,
-
+
case couch_doc:has_stubs(Doc) of
true ->
DiskDoc = LoadPrevRevFun(),
@@ -632,7 +641,7 @@ prep_and_validate_replicated_updates(Db, [Bucket|RestBuckets], [OldInfo|RestOldI
Doc2 = Doc,
GetDiskDocFun = LoadPrevRevFun
end,
-
+
case validate_doc_update(Db, Doc2, GetDiskDocFun) of
ok ->
{[Doc2 | AccValidated], AccErrors2};
@@ -727,7 +736,7 @@ update_docs(Db, Docs, Options, interactive_edit) ->
{[Doc | DocsAcc], NonRepDocsAcc}
end
end, {[], []}, Docs),
-
+
DocBuckets = group_alike_docs(Docs2),
case (Db#db.validate_doc_funs /= []) orelse
@@ -767,9 +776,9 @@ update_docs(Db, Docs, Options, interactive_edit) ->
check_dup_atts(Doc)), Db#db.fd)
|| Doc <- B] || B <- DocBuckets2],
{DocBuckets4, IdRevs} = new_revs(DocBuckets3, [], []),
-
+
{ok, CommitResults} = write_and_commit(Db, DocBuckets4, NonRepDocs, Options2),
-
+
ResultsDict = dict:from_list(IdRevs ++ CommitResults ++ PreCommitFailures),
{ok, lists:map(
fun(#doc{id=Id,revs={Pos, RevIds}}) ->
@@ -855,7 +864,7 @@ set_new_att_revpos(#doc{revs={RevPos,_Revs},atts=Atts}=Doc) ->
(Att) ->
Att#att{revpos=RevPos+1}
end, Atts)}.
-
+
doc_flush_atts(Doc, Fd) ->
Doc#doc{atts=[flush_att(Fd, Att) || Att <- Doc#doc.atts]}.
@@ -986,13 +995,15 @@ enum_docs_since_reduce_to_count(Reds) ->
fun couch_db_updater:btree_by_seq_reduce/2, Reds).
enum_docs_reduce_to_count(Reds) ->
- {Count, _, _} = couch_btree:final_reduce(
- fun couch_db_updater:btree_by_id_reduce/2, Reds),
- Count.
+ case couch_btree:final_reduce(
+ fun couch_db_updater:btree_by_id_reduce/2, Reds) of
+ {C1, _, _} -> C1;
+ {C2, _, _, _} -> C2
+ end.
changes_since(Db, StartSeq, Fun, Acc) ->
changes_since(Db, StartSeq, Fun, [], Acc).
-
+
changes_since(Db, StartSeq, Fun, Options, Acc) ->
Wrapper = fun(FullDocInfo, _Offset, Acc2) ->
case FullDocInfo of
View
40 apps/couch/src/couch_db_updater.erl
@@ -362,23 +362,37 @@ btree_by_id_join(Id, {HighSeq, Deleted, Size, DiskTree}) ->
btree_by_id_reduce(reduce, FullDocInfos) ->
lists:foldl(
- fun(#full_doc_info{deleted = false, data_size=Size},
- {NotDeleted, Deleted, DocSize}) ->
- {NotDeleted + 1, Deleted, DocSize + Size};
- (#full_doc_info{deleted = true, data_size=Size},
- {NotDeleted, Deleted, DocSize}) ->
- {NotDeleted, Deleted + 1, DocSize + Size}
+ fun(#full_doc_info{deleted = false, rev_tree = Tree, data_size=Size},
+ {NotDeleted, Deleted, Conflicts, DocSize}) ->
+ {NotDeleted + 1, Deleted,
+ Conflicts + case has_conflicts(Tree) of
+ true -> 1;
+ false -> 0
+ end,
+ DocSize + Size};
+ (#full_doc_info{deleted = true, rev_tree = Tree, data_size=Size},
+ {NotDeleted, Deleted, Conflicts, DocSize}) ->
+ {NotDeleted, Deleted + 1,
+ Conflicts + case has_conflicts(Tree) of
+ true -> 1;
+ false -> 0
+ end,
+ DocSize + Size}
end,
- {0, 0, 0}, FullDocInfos);
+ {0, 0, 0, 0}, FullDocInfos);
-btree_by_id_reduce(rereduce, Reductions) ->
+btree_by_id_reduce(rereduce, [FirstRed | RestReds]) ->
lists:foldl(
fun({NotDeleted, Deleted}, {AccNotDeleted, AccDeleted, AccDocSizes}) ->
- {AccNotDeleted + NotDeleted, AccDeleted + Deleted, AccDocSizes};
- ({NotDeleted, Deleted, DocSizes}, {AccNotDeleted, AccDeleted, AccDocSizes}) ->
- {AccNotDeleted + NotDeleted, AccDeleted + Deleted, DocSizes + AccDocSizes}
+ {AccNotDeleted + NotDeleted, AccDeleted + Deleted, 0, AccDocSizes};
+ ({NotDeleted, Deleted, DocSizes}, {AccNotDeleted, AccDeleted, AccConflicts, AccDocSizes}) ->
+ {AccNotDeleted + NotDeleted, AccDeleted + Deleted, AccConflicts, DocSizes + AccDocSizes};
+ ({NotDeleted, Deleted, Conflicts, DocSizes},
+ {AccNotDeleted, AccDeleted, AccConflicts, AccDocSizes}) ->
+ {AccNotDeleted + NotDeleted, AccDeleted + Deleted,
+ Conflicts + AccConflicts, DocSizes + AccDocSizes}
end,
- {0, 0, 0}, Reductions).
+ FirstRed, RestReds).
btree_by_seq_reduce(reduce, DocInfos) ->
% count the number of documents
@@ -458,6 +472,8 @@ init_db(DbName, Filepath, Fd, Header0) ->
close_db(#db{fd_monitor = Ref}) ->
erlang:demonitor(Ref).
+has_conflicts(RevTree) ->
+ couch_key_tree:has_conflicts(RevTree).
refresh_validate_doc_funs(Db) ->
{ok, DesignDocs} = couch_db:get_design_docs(Db),
View
45 apps/couch/src/couch_key_tree.erl
@@ -50,7 +50,7 @@
-export([merge/3, find_missing/2, get_key_leafs/2,
get_full_key_paths/2, get/2, compute_data_size/1]).
-export([map/2, get_all_leafs/1, count_leafs/1, remove_leafs/2,
- get_all_leafs_full/1,stem/2,map_leafs/2, fold/3]).
+ has_conflicts/1, get_all_leafs_full/1,stem/2,map_leafs/2, fold/3]).
-include("couch_db.hrl").
@@ -325,6 +325,49 @@ count_leafs_simple([{_Key, _Value, []} | RestTree]) ->
count_leafs_simple([{_Key, _Value, SubTree} | RestTree]) ->
count_leafs_simple(SubTree) + count_leafs_simple(RestTree).
+%% @doc check a revision tree for conflicts. By definition a tree has
+%% conflicts if there is more than one leaf that is not deleted
+has_conflicts([]) ->
+ false;
+has_conflicts(BranchList) ->
+ count_non_deleted(BranchList,0) > 1.
+
+count_non_deleted([],N) ->
+ N;
+count_non_deleted([{_Pos, Tree} | RestTree], N) ->
+ % once there is more than one return
+ case N > 1 of
+ true -> N;
+ false ->
+ NextN = count_non_deleted_leaves([Tree],N),
+ count_non_deleted(RestTree, NextN)
+ end.
+
+count_non_deleted_leaves([],N) ->
+ N;
+count_non_deleted_leaves([{_Key, Value, []} | RestTree],N) ->
+ NextN = case check_deleted(Value) of false -> N + 1; true -> N end,
+ case NextN > 1 of
+ true -> NextN;
+ false ->
+ count_non_deleted_leaves(RestTree, NextN)
+ end;
+count_non_deleted_leaves([{_Key, _Value, SubTree} | RestTree],N) ->
+ NextN = count_non_deleted_leaves(SubTree,N),
+ case NextN > 1 of
+ true ->
+ NextN;
+ false ->
+ count_non_deleted_leaves(RestTree, NextN)
+ end.
+
+check_deleted({true,_,_}) ->
+ true;
+check_deleted(#doc{deleted=true}) ->
+ true;
+check_deleted(_) ->
+ false.
+
compute_data_size(Tree) ->
{TotBodySizes,TotAttSizes} =
tree_fold(fun({_Pos, _Key, _Value},branch,Acc) ->
View
6 apps/couch/src/couch_view.erl
@@ -118,8 +118,10 @@ list_index_files(Db) ->
get_row_count(#view{btree=Bt}) ->
- {ok, {Count, _, _}} = couch_btree:full_reduce(Bt),
- {ok, Count}.
+ case couch_btree:full_reduce(Bt) of
+ {ok, {NC, _, _, _}} -> NC;
+ {ok, {OC, _, _}} -> OC
+ end.
get_temp_reduce_view(Db, Language, DesignOptions, MapSrc, RedSrc) ->
{ok, #group{views=[View]}=Group} =
View
8 apps/couch/src/couch_view_group.erl
@@ -530,8 +530,12 @@ get_group_info(State) ->
compute_data_size(ViewList) ->
lists:foldl(fun(#view{btree=Btree}, Acc) ->
- {ok, {_, _, Size}} = couch_btree:full_reduce(Btree),
- Size + Acc
+ case couch_btree:full_reduce(Btree) of
+ {ok, {_, _, _, NSize}} ->
+ NSize + Acc;
+ {ok, {_, _, OSize}} ->
+ OSize + Acc
+ end
end, 0, ViewList).
Something went wrong with that request. Please try again.