Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
Merge pull request #33 from cloudant/11602-sync-security
BugzID: 11602
  • Loading branch information
kocolosk committed Oct 1, 2012
2 parents b239d17 + 32578e4 commit bd4b89717315f6550b56a141fb771b147440a0d5
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 20 deletions.
@@ -10,5 +10,5 @@
mem3_sync_nodes,
mem3_sup
]},
{applications, [kernel, stdlib, sasl, crypto, mochiweb, couch, twig]}
{applications, [kernel, stdlib, sasl, crypto, mochiweb, couch, rexi, twig]}
]}.
@@ -26,6 +26,7 @@ go(DbName, Node, Opts) when is_binary(DbName), is_atom(Node) ->
go(#shard{name=DbName, node=node()}, #shard{name=DbName, node=Node}, Opts);

go(#shard{} = Source, #shard{} = Target, Opts) ->
mem3_sync_security:maybe_sync(Source, Target),
BatchSize = case proplists:get_value(batch_size, Opts) of
BS when is_integer(BS), BS > 0 -> BS;
_ -> 100
@@ -38,12 +38,9 @@ handle_call(_Request, State) ->
{ok, ok, State}.

handle_info({nodeup, Node}, State) ->
case lists:member(Node, mem3:nodes()) of
true ->
mem3_sync_nodes:add([Node]);
false ->
ok
end,
Nodes0 = lists:usort(drain_nodeups([Node])),
Nodes = lists:filter(fun(N) -> lists:member(N, mem3:nodes()) end, Nodes0),
wait_for_rexi(Nodes, 5),
{ok, State};

handle_info({nodedown, Node}, State) ->
@@ -58,3 +55,33 @@ terminate(_Reason, _State) ->

code_change(_OldVsn, State, _Extra) ->
{ok, State}.

drain_nodeups(Acc) ->
receive
{nodeup, Node} ->
drain_nodeups([Node | Acc])
after 0 ->
Acc
end.

wait_for_rexi([], _Retries) ->
ok;
wait_for_rexi(Waiting, Retries) ->
% Hack around rpc:multicall/4 so that we can
% be sure which nodes gave which response
Msg = {call, erlang, whereis, [rexi_server], group_leader()},
{Resp, _Bad} = gen_server:multi_call(Waiting, rex, Msg, 1000),
Up = [N || {N, P} <- Resp, is_pid(P)],
NotUp = Waiting -- Up,
case length(Up) > 0 of
true ->
mem3_sync_nodes:add(Up);
false -> ok
end,
case length(NotUp) > 0 andalso Retries > 0 of
true ->
timer:sleep(1000),
wait_for_rexi(NotUp, Retries-1);
false ->
ok
end.
@@ -14,8 +14,34 @@

-module(mem3_sync_security).

-export([maybe_sync/2, maybe_sync_int/2]).
-export([go/0, go/1]).

-include("mem3.hrl").


maybe_sync(#shard{}=Src, #shard{}=Dst) ->
case is_local(Src#shard.name) of
false ->
erlang:spawn(?MODULE, maybe_sync_int, [Src, Dst]);
true ->
ok
end.

maybe_sync_int(#shard{name=Name}=Src, Dst) ->
DbName = mem3:dbname(Name),
case fabric:get_all_security(DbName, [{shards, [Src, Dst]}]) of
{ok, WorkerObjs} ->
Objs = [Obj || {_Worker, Obj} <- WorkerObjs],
case length(lists:usort(Objs)) of
1 -> ok;
2 -> go(DbName)
end;
Else ->
Args = [DbName, Else],
twig:log(err, "Error checking security objects for ~s :: ~p", Args)
end.

go() ->
{ok, Dbs} = fabric:all_dbs(),
lists:foreach(fun handle_db/1, Dbs).
@@ -24,9 +50,10 @@ go(DbName) when is_binary(DbName) ->
handle_db(DbName).

handle_db(DbName) ->
ShardCount = length(mem3:shards(DbName)),
case get_all_security(DbName) of
{ok, SecObjs} ->
case is_ok(SecObjs) of
case is_ok(SecObjs, ShardCount) of
ok ->
ok;
{fixable, SecObj} ->
@@ -56,21 +83,25 @@ get_all_security(DbName) ->
Error
end.

is_ok([_]) ->
is_ok([_], _) ->
% One security object is the happy case
ok;
is_ok([_, _] = SecObjs0) ->
% This is the strict heuristic where there is one version of
% the security object that out numbers empty security objects.
% If so, just overwrite the empty objects with the non-empty
% version.
case lists:keytake({[]}, 1, SecObjs0) of
{value, {_, EmptyCount}, [{SecObj, Count}]} when Count > EmptyCount ->
{fixable, SecObj};
_ ->
broken
is_ok([_, _] = SecObjs0, ShardCount) ->
% Figure out if we have a simple majority of security objects
% and if so, use that as the correct value. Otherwise we abort
% and rely on human intervention.
{Count, SecObj} = lists:max([{C, O} || {O, C} <- SecObjs0]),
case Count >= ((ShardCount div 2) + 1) of
true -> {fixable, SecObj};
false -> broken
end;
is_ok(_) ->
is_ok(_, _) ->
% Anything else requires human intervention
broken.


is_local(<<"shards/", _/binary>>) ->
false;
is_local(_) ->
true.

0 comments on commit bd4b897

Please sign in to comment.