Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Further progress, but still unhealthy service no worky

  • Loading branch information...
commit e2614132d44b80de4e247176c783d32c6a943876 1 parent a6e4ecf
@lordnull lordnull authored
Showing with 56 additions and 30 deletions.
  1. +11 −2 src/riak_core_node_watcher.erl
  2. +45 −28 test/node_watcher_qc.erl
View
13 src/riak_core_node_watcher.erl
@@ -597,7 +597,7 @@ handle_health_check_exit(Id, false, #health_check{health_failures = N, max_healt
CheckRec2 = CheckRec#health_check{health_failures = N + 1, checking_pid = undefined, callback_failures = 0},
{reply, _, State2} = handle_call({service_down, Id}, "from", State),
CheckRec3 = schedule_health_check(Id, CheckRec2),
- Checks = orddict:store(Id, CheckRec3),
+ Checks = orddict:store(Id, CheckRec3, State2#state.health_checks),
State2#state{health_checks = Checks};
% healthy failure while service_down
@@ -657,6 +657,15 @@ trigger_health_check(Id, Rec, true) ->
erlang:cancel_timer(Rec#health_check.interval_tref),
#health_check{callback = MFA, service_pid = ServPid} = Rec,
{Mod, Fun, Args} = MFA,
- Pid = proc_lib:spawn_link(Mod, Fun, [ServPid | Args]),
+ CheckFun = fun() ->
+ Out = erlang:apply(Mod,Fun,[ServPid | Args]),
+ if
+ Out ->
+ ok;
+ true ->
+ exit(Out)
+ end
+ end,
+ Pid = proc_lib:spawn_link(CheckFun),
erlang:put(Pid, Id),
Rec#health_check{checking_pid = Pid, interval_tref = undefined}.
View
73 test/node_watcher_qc.erl
@@ -96,21 +96,21 @@ initial_state() ->
command(S) ->
oneof([
- {call, ?MODULE, ring_update, [g_ring_nodes()]},
- {call, ?MODULE, local_service_up, [g_service()]},
- {call, ?MODULE, local_service_down, [g_service()]},
- {call, ?MODULE, local_service_kill, [g_service(), S]},
- {call, ?MODULE, local_node_up, []},
- {call, ?MODULE, local_node_down, []},
- {call, ?MODULE, remote_service_up, [g_node(), g_services()]},
- {call, ?MODULE, remote_service_down, [g_node()]},
- {call, ?MODULE, remote_service_down_disterl, [g_node()]},
- {call, ?MODULE, wait_for_bcast, []},
- {call, ?MODULE, healthy_service, [g_service()]},
- {call, ?MODULE, unhealthy_service, [g_service()]},
- {call, ?MODULE, recovering_service, [g_service()]},
- {call, ?MODULE, faulty_health_check_callback_once, [g_service()]},
- {call, ?MODULE, faulty_health_check_callback_many, [g_service()]}
+ %{call, ?MODULE, ring_update, [g_ring_nodes()]},
+ %{call, ?MODULE, local_service_up, [g_service()]},
+ %{call, ?MODULE, local_service_down, [g_service()]},
+ %{call, ?MODULE, local_service_kill, [g_service(), S]},
+ %{call, ?MODULE, local_node_up, []},
+ %{call, ?MODULE, local_node_down, []},
+ %{call, ?MODULE, remote_service_up, [g_node(), g_services()]},
+ %{call, ?MODULE, remote_service_down, [g_node()]},
+ %{call, ?MODULE, remote_service_down_disterl, [g_node()]},
+ %{call, ?MODULE, wait_for_bcast, []},
+ %{call, ?MODULE, healthy_service, [g_service()]},
+ {call, ?MODULE, unhealthy_service, [g_service()]}%,
+ %{call, ?MODULE, recovering_service, [g_service()]},
+ %{call, ?MODULE, faulty_health_check_callback_once, [g_service()]},
+ %{call, ?MODULE, faulty_health_check_callback_many, [g_service()]}
]).
precondition(S, {call, _, local_service_kill, [Service, S]}) ->
@@ -158,10 +158,15 @@ next_state(S, Res, {call, _, healthy_service, [Service]}) ->
S2#state { service_pids = Pids };
next_state(S, Res, {call, M, unhealthy_service, [Service]}) ->
- meck:expect(mod_health, callback, fun (Pid, false) when is_pid(Pid) ->
- false
- end),
- next_state(S, Res, {call, M, local_service_up, [Service]});
+ ?debugFmt("Next state:\n"
+ " in state: ~p\n"
+ " last res: ~p\n"
+ " service: ~p",
+ [S,Res,Service]),
+ S;
+ %S2 = service_up(node(), Service, S),
+ %Pids = orddict:store(Service, Res, S2#state.service_pids),
+ %S2#state { service_pids = Pids };
next_state(S, Res, {call, M, recovering_service, [Service]}) ->
meck:sequence(mod_health, callback, 2, [false, true]),
@@ -255,16 +260,17 @@ postcondition(S, {call, _, healthy_service, [Service]}, _Res) ->
?assert(meck:validate(mod_health)),
deep_validate(S2);
-postcondition(S, {call, _, healthy_existing_service, [Service]}, _Res) ->
+postcondition(S, {call, _, unhealthy_service, [Service]}, Res) ->
+ ?debugFmt("postcondition:\n"
+ " in state: ~p\n"
+ " service: ~p\n"
+ " in res: ~p",
+ [S, Service, Res]),
S2 = service_up(node(), Service, S),
+ S3 = service_down(node(), Service, S2),
+ validate_broadcasts([S,S2,S3], Service),
?assert(meck:validate(mod_health)),
- deep_validate(S2);
-
-postcondition(S, {call, _, unhealthy_service, [Service]}, _Res) ->
- S2 = service_down(node(), Service, S),
- validate_broadcast(S, S2, service),
- ?assert(meck:validate(mod_health)),
- deep_validate(S2);
+ deep_validate(S3);
postcondition(S, {call, _, recovering_service, [Service]}, _Res) ->
S2 = service_down(node(), Service, S),
@@ -338,6 +344,7 @@ validate_broadcast(S0, Sfinal, Op, Bcasts) ->
validate_broadcasts(States, Op) ->
Bcasts = broadcasts(),
+ ?debugFmt("broadcasts: ~p", [Bcasts]),
?assertEqual(length(Bcasts) + 1, length(States)),
validate_broadcasts(States, Op, Bcasts).
@@ -440,9 +447,19 @@ healthy_service(Service) ->
unhealthy_service(Service) ->
Pid = spawn(fun() -> service_loop() end),
- ok = riak_core_node_watcher:service_up(Service, Pid),
+ Self = self(),
+ meck:expect(mod_health, callback, fun (InPid, false) when is_pid(InPid) ->
+ Self ! {meck_done, InPid},
+ false
+ end),
ok = riak_core_node_watcher:service_up(Service, Pid, {mod_health, callback, [false]}),
gen_server:cast(riak_core_node_watcher, {force_health_check, Service}),
+ receive
+ {meck_done, MeckInPid} ->
+ ?assertEqual(Pid, MeckInPid)
+ after 1000 ->
+ erlang:error(timeout)
+ end,
Pid.
recovering_service(Service) ->
Please sign in to comment.
Something went wrong with that request. Please try again.