Permalink
Browse files

Fix handoff issues in vnode/vnode manager

Change trigger_handoff and finish_handoff events in the
vnode to be ignored if the vnode modstate is already deleted.

Add periodic timer to vnode manager to perform various management
activities. Currently re-triggers ownership handoff so that handoff
under load is not triggered only on ring changes.
  • Loading branch information...
1 parent d347ece commit 03d74c208a65bfeeaef582bf9472e58d997d5276 @jtuple jtuple committed Dec 23, 2011
Showing with 39 additions and 16 deletions.
  1. +11 −4 src/riak_core_vnode.erl
  2. +28 −12 src/riak_core_vnode_manager.erl
View
15 src/riak_core_vnode.erl
@@ -398,6 +398,10 @@ handle_event({set_forwarding, ForwardTo}, _StateName, State) ->
lager:debug("vnode fwd :: ~p/~p :: ~p -> ~p~n",
[State#state.mod, State#state.index, State#state.forward, ForwardTo]),
continue(State#state{forward=ForwardTo});
+handle_event(finish_handoff, _StateName,
+ State=#state{modstate={deleted, _ModState}}) ->
+ stop_manager_event_timer(State),
+ continue(State#state{handoff_node=none});
handle_event(finish_handoff, _StateName, State=#state{mod=Mod,
modstate=ModState,
handoff_node=HN}) ->
@@ -406,8 +410,8 @@ handle_event(finish_handoff, _StateName, State=#state{mod=Mod,
none ->
continue(State);
_ ->
- Mod:handoff_finished(HN, ModState),
- finish_handoff(State)
+ {ok, NewModState} = Mod:handoff_finished(HN, ModState),
+ finish_handoff(State#state{modstate=NewModState})
end;
handle_event(cancel_handoff, _StateName, State=#state{mod=Mod,
modstate=ModState}) ->
@@ -418,9 +422,12 @@ handle_event(cancel_handoff, _StateName, State=#state{mod=Mod,
none ->
continue(State);
_ ->
- Mod:handoff_cancelled(ModState),
- continue(State#state{handoff_node=none})
+ {ok, NewModState} = Mod:handoff_cancelled(ModState),
+ continue(State#state{handoff_node=none, modstate=NewModState})
end;
+handle_event({trigger_handoff, _TargetNode}, _StateName,
+ State=#state{modstate={deleted, _ModState}}) ->
+ continue(State);
handle_event(R={trigger_handoff, _TargetNode}, _StateName, State) ->
active(R, State);
handle_event(R=?VNODE_REQ{}, _StateName, State) ->
View
40 src/riak_core_vnode_manager.erl
@@ -107,6 +107,7 @@ init(_State) ->
AllVNodes = get_all_vnodes(Mods, State2),
State3 = update_forwarding(AllVNodes, Mods, Ring, State2),
State4 = update_handoff(AllVNodes, Ring, State3),
+ schedule_management_timer(),
{ok, State4}.
%% @private
@@ -196,21 +197,19 @@ handle_cast({ring_changed, Ring}, State) ->
State3 = update_handoff(AllVNodes, Ring, State2),
%% Trigger ownership transfers.
- Transfers = riak_core_ring:pending_changes(Ring),
- Limit = app_helper:get_env(riak_core,
- forced_ownership_handoff,
- ?DEFAULT_OWNERSHIP_TRIGGER),
- Throttle = lists:sublist(Transfers, Limit),
- Awaiting = [{Mod, Idx} || {Idx, Node, _, CMods, S} <- Throttle,
- Mod <- Mods,
- S =:= awaiting,
- Node =:= node(),
- not lists:member(Mod, CMods)],
-
- [maybe_trigger_handoff(Mod, Idx, State3) || {Mod, Idx} <- Awaiting],
+ trigger_ownership_handoff(Mods, Ring, State3),
{noreply, State3};
+handle_cast(management_tick, State) ->
+ schedule_management_timer(),
+ {ok, Ring} = riak_core_ring_manager:get_raw_ring(),
+ Mods = [Mod || {_, Mod} <- riak_core:vnode_modules()],
+ AllVNodes = get_all_vnodes(Mods, State),
+ State2 = update_handoff(AllVNodes, Ring, State),
+ trigger_ownership_handoff(Mods, Ring, State2),
+ {noreply, State2};
+
handle_cast(_, State) ->
{noreply, State}.
@@ -244,6 +243,23 @@ code_change(_OldVsn, State, _Extra) ->
%% Internal functions
%% ===================================================================
+schedule_management_timer() ->
+ timer:apply_after(30000, gen_server, cast, [?MODULE, management_tick]).
+
+trigger_ownership_handoff(Mods, Ring, State) ->
+ Transfers = riak_core_ring:pending_changes(Ring),
+ Limit = app_helper:get_env(riak_core,
+ forced_ownership_handoff,
+ ?DEFAULT_OWNERSHIP_TRIGGER),
+ Throttle = lists:sublist(Transfers, Limit),
+ Awaiting = [{Mod, Idx} || {Idx, Node, _, CMods, S} <- Throttle,
+ Mod <- Mods,
+ S =:= awaiting,
+ Node =:= node(),
+ not lists:member(Mod, CMods)],
+ [maybe_trigger_handoff(Mod, Idx, State) || {Mod, Idx} <- Awaiting],
+ ok.
+
get_all_index_pid(Mod, State) ->
[list_to_tuple(L)
|| L <- ets:match(State#state.idxtab, {idxrec, '_', '$1', Mod, '$2', '_'})].

0 comments on commit 03d74c2

Please sign in to comment.