Permalink
Cannot retrieve contributors at this time
%-*-Mode:erlang;coding:utf-8;tab-width:4;c-basic-offset:4;indent-tabs-mode:()-*- | |
% ex: set ft=erlang fenc=utf-8 sts=4 ts=4 sw=4 et nomod: | |
%%% | |
%%%------------------------------------------------------------------------ | |
%%% @doc | |
%%% ==CloudI Quorum Service== | |
%%% Using this service can provide Byzantine fault tolerance for any other | |
%%% services. The service prefix is used to accept any service requests | |
%%% that match the prefix, while the suffix is used to achieve the | |
%%% configured number of service request responses | |
%%% (i.e., quorum: the minimum number of services able to process a | |
%%% service request to achieve the same response). If quorum is met, | |
%%% the agreed upon response is returned. Otherwise, the service request to | |
%%% this service will timeout (i.e., receive no response). | |
%%% @end | |
%%% | |
%%% MIT License | |
%%% | |
%%% Copyright (c) 2013-2019 Michael Truog <mjtruog at protonmail dot com> | |
%%% | |
%%% Permission is hereby granted, free of charge, to any person obtaining a | |
%%% copy of this software and associated documentation files (the "Software"), | |
%%% to deal in the Software without restriction, including without limitation | |
%%% the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
%%% and/or sell copies of the Software, and to permit persons to whom the | |
%%% Software is furnished to do so, subject to the following conditions: | |
%%% | |
%%% The above copyright notice and this permission notice shall be included in | |
%%% all copies or substantial portions of the Software. | |
%%% | |
%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
%%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | |
%%% DEALINGS IN THE SOFTWARE. | |
%%% | |
%%% @author Michael Truog <mjtruog at protonmail dot com> | |
%%% @copyright 2013-2019 Michael Truog | |
%%% @version 1.8.0 {@date} {@time} | |
%%%------------------------------------------------------------------------ | |
-module(cloudi_service_quorum). | |
-author('mjtruog at protonmail dot com'). | |
-behaviour(cloudi_service). | |
%% external interface | |
%% cloudi_service callbacks | |
-export([cloudi_service_init/4, | |
cloudi_service_handle_request/11, | |
cloudi_service_handle_info/3, | |
cloudi_service_terminate/3]). | |
-include_lib("cloudi_core/include/cloudi_logger.hrl"). | |
-include_lib("cloudi_core/include/cloudi_service.hrl"). | |
-define(DEFAULT_QUORUM, byzantine). % see below: | |
% quorum is specified either as an absolute integer or percentage, | |
% or as 'byzantine' for (N - floor((N - 1) / 3)) | |
% ('byzantine' requires that less than 1/3rd of the | |
% responses be erroneous (e.g., a timeout or a non-matching response)) | |
-define(DEFAULT_USE_RESPONSE_INFO, true). % match response_info | |
-define(DEFAULT_VALIDATE_REQUEST_INFO, undefined). | |
-define(DEFAULT_VALIDATE_REQUEST, undefined). | |
-define(DEFAULT_VALIDATE_RESPONSE_INFO, undefined). | |
-define(DEFAULT_VALIDATE_RESPONSE, | |
fun | |
(<<>>, <<>>) -> | |
false; | |
(_, _) -> | |
true | |
end). | |
-define(DEFAULT_FAILURES_SOURCE_DIE, false). | |
-define(DEFAULT_FAILURES_SOURCE_MAX_COUNT, 2). % see below: | |
% (similar to the MaxR configuration value for services) | |
-define(DEFAULT_FAILURES_SOURCE_MAX_PERIOD, 60). % seconds, see below: | |
% (similar to the MaxT configuration value for services) | |
% If you want the source service to eventually fail, | |
% use the service's MaxT/MaxR as the failures_source_max_period value | |
% (e.g., 300/5 == 60 seconds). Can also use the value 'infinity' | |
% to accumulate a failure count indefinitely. | |
-define(DEFAULT_FAILURES_DEST_DIE, false). | |
-define(DEFAULT_FAILURES_DEST_MAX_COUNT, 2). % see below: | |
% (similar to the MaxR configuration value for services) | |
-define(DEFAULT_FAILURES_DEST_MAX_PERIOD, 60). % seconds, see below: | |
% (similar to the MaxT configuration value for services) | |
% If you want the destination service to eventually fail, | |
% use the service's MaxT/MaxR as the failures_dest_max_period value | |
% (e.g., 300/5 == 60 seconds). Can also use the value 'infinity' | |
% to accumulate a failure count indefinitely. | |
-record(request, | |
{ | |
request_type :: cloudi_service:request_type(), | |
name :: cloudi_service:service_name(), | |
pattern :: cloudi_service:service_name_pattern(), | |
timeout :: cloudi_service:timeout_value_milliseconds(), | |
source :: cloudi_service:source(), | |
count_required :: pos_integer(), | |
count_total :: pos_integer(), | |
count_responses = 0 :: non_neg_integer(), | |
count_correct = 0 :: non_neg_integer(), | |
responses = [] | |
:: list({{cloudi_service:response_info(), | |
cloudi_service:response()} | cloudi_service:response(), | |
pos_integer()}), | |
returned :: boolean() | |
}). | |
-record(pending, | |
{ | |
trans_id :: cloudi_service:trans_id(), | |
dest :: pid() | |
}). | |
-record(state, | |
{ | |
quorum :: byzantine | number(), | |
use_response_info :: boolean(), | |
validate_request_info :: undefined | fun((any()) -> boolean()), | |
validate_request :: undefined | fun((any(), any()) -> boolean()), | |
validate_response_info :: undefined | fun((any()) -> boolean()), | |
validate_response :: undefined | fun((any(), any()) -> boolean()), | |
failures_source_die :: boolean(), | |
failures_source_max_count :: pos_integer(), | |
failures_source_max_period :: infinity | pos_integer(), | |
failures_source = #{} | |
:: #{pid() := list(cloudi_timestamp:seconds_monotonic())}, | |
failures_dest_die :: boolean(), | |
failures_dest_max_count :: pos_integer(), | |
failures_dest_max_period :: infinity | pos_integer(), | |
failures_dest = #{} | |
:: #{pid() := list(cloudi_timestamp:seconds_monotonic())}, | |
requests = #{} | |
:: #{cloudi_service:trans_id() := #request{}}, % orig | |
pending = #{} | |
:: #{cloudi_service:trans_id() := % new | |
cloudi_service:trans_id()} % orig | |
}). | |
%%%------------------------------------------------------------------------ | |
%%% External interface functions | |
%%%------------------------------------------------------------------------ | |
%%%------------------------------------------------------------------------ | |
%%% Callback functions from cloudi_service | |
%%%------------------------------------------------------------------------ | |
cloudi_service_init(Args, Prefix, _Timeout, Dispatcher) -> | |
Defaults = [ | |
{quorum, ?DEFAULT_QUORUM}, | |
{use_response_info, ?DEFAULT_USE_RESPONSE_INFO}, | |
{validate_request_info, ?DEFAULT_VALIDATE_REQUEST_INFO}, | |
{validate_request, ?DEFAULT_VALIDATE_REQUEST}, | |
{validate_response_info, ?DEFAULT_VALIDATE_RESPONSE_INFO}, | |
{validate_response, ?DEFAULT_VALIDATE_RESPONSE}, | |
{failures_source_die, ?DEFAULT_FAILURES_SOURCE_DIE}, | |
{failures_source_max_count, ?DEFAULT_FAILURES_SOURCE_MAX_COUNT}, | |
{failures_source_max_period, ?DEFAULT_FAILURES_SOURCE_MAX_PERIOD}, | |
{failures_dest_die, ?DEFAULT_FAILURES_DEST_DIE}, | |
{failures_dest_max_count, ?DEFAULT_FAILURES_DEST_MAX_COUNT}, | |
{failures_dest_max_period, ?DEFAULT_FAILURES_DEST_MAX_PERIOD}], | |
[Quorum, UseResponseInfo, | |
ValidateRequestInfo0, ValidateRequest0, | |
ValidateResponseInfo0, ValidateResponse0, | |
FailuresSrcDie, FailuresSrcMaxCount, FailuresSrcMaxPeriod, | |
FailuresDstDie, FailuresDstMaxCount, FailuresDstMaxPeriod | |
] = cloudi_proplists:take_values(Defaults, Args), | |
true = ((is_atom(Quorum) andalso | |
(Quorum =:= byzantine)) orelse | |
(is_float(Quorum) andalso | |
(Quorum > 0.0) andalso (Quorum =< 1.0)) orelse | |
(is_integer(Quorum) andalso | |
(Quorum > 0))), | |
true = is_boolean(UseResponseInfo), | |
ValidateRequestInfo1 = cloudi_args_type: | |
function_optional(ValidateRequestInfo0, 1), | |
ValidateRequest1 = cloudi_args_type: | |
function_optional(ValidateRequest0, 2), | |
ValidateResponseInfo1 = cloudi_args_type: | |
function_optional(ValidateResponseInfo0, 1), | |
ValidateResponse1 = cloudi_args_type: | |
function_optional(ValidateResponse0, 2), | |
true = is_boolean(FailuresSrcDie), | |
true = is_integer(FailuresSrcMaxCount) andalso (FailuresSrcMaxCount > 0), | |
true = (FailuresSrcMaxPeriod =:= infinity) orelse | |
(is_integer(FailuresSrcMaxPeriod) andalso | |
(FailuresSrcMaxPeriod > 0)), | |
true = is_boolean(FailuresDstDie), | |
true = is_integer(FailuresDstMaxCount) andalso (FailuresDstMaxCount > 0), | |
true = (FailuresDstMaxPeriod =:= infinity) orelse | |
(is_integer(FailuresDstMaxPeriod) andalso | |
(FailuresDstMaxPeriod > 0)), | |
false = cloudi_service_name:pattern(Prefix), | |
cloudi_service:subscribe(Dispatcher, "*"), | |
{ok, #state{quorum = Quorum, | |
use_response_info = UseResponseInfo, | |
validate_request_info = ValidateRequestInfo1, | |
validate_request = ValidateRequest1, | |
validate_response_info = ValidateResponseInfo1, | |
validate_response = ValidateResponse1, | |
failures_source_die = FailuresSrcDie, | |
failures_source_max_count = FailuresSrcMaxCount, | |
failures_source_max_period = FailuresSrcMaxPeriod, | |
failures_dest_die = FailuresDstDie, | |
failures_dest_max_count = FailuresDstMaxCount, | |
failures_dest_max_period = FailuresDstMaxPeriod}}. | |
cloudi_service_handle_request(RequestType, Name, Pattern, RequestInfo, Request, | |
Timeout, Priority, TransId, SrcPid, | |
#state{quorum = Quorum, | |
validate_request_info = RequestInfoF, | |
validate_request = RequestF} = State, | |
Dispatcher) -> | |
case validate(RequestInfoF, RequestF, | |
RequestInfo, Request) of | |
true -> | |
[QuorumName] = cloudi_service_name:parse(Name, Pattern), | |
case cloudi_service:get_pids(Dispatcher, QuorumName, Timeout) of | |
{ok, PatternPids} -> | |
Count = erlang:length(PatternPids), | |
CountRequired = if | |
Quorum =:= byzantine -> | |
if | |
Count < 4 -> | |
?LOG_ERROR("Byzantine quorum not met!" | |
" ~p N=~p", [QuorumName, Count]), | |
undefined; | |
true -> | |
Count - cloudi_math:floor((Count - 1) / 3) | |
end; | |
is_integer(Quorum) -> | |
if | |
Quorum > Count -> | |
?LOG_ERROR("Absolute quorum not met!" | |
" ~p N=~p", [QuorumName, Count]), | |
undefined; | |
true -> | |
Quorum | |
end; | |
is_float(Quorum) -> | |
erlang:min(Count, | |
cloudi_math:ceil(Quorum * Count)) | |
end, | |
if | |
CountRequired =:= undefined -> | |
request_failed(SrcPid, State); | |
true -> | |
mcast(PatternPids, QuorumName, | |
RequestType, Name, Pattern, | |
RequestInfo, Request, | |
Timeout, Priority, TransId, SrcPid, | |
CountRequired, State, Dispatcher) | |
end; | |
{error, timeout} -> | |
request_failed(SrcPid, State) | |
end; | |
false -> | |
request_failed(SrcPid, State) | |
end. | |
cloudi_service_handle_info(#return_async_active{response_info = ResponseInfo, | |
response = Response, | |
timeout = Timeout, | |
trans_id = QuorumTransId}, | |
#state{use_response_info = UseResponseInfo, | |
validate_response_info = ResponseInfoF, | |
validate_response = ResponseF, | |
failures_source_die = FailuresSrcDie, | |
failures_source_max_count = | |
FailuresSrcMaxCount, | |
failures_source_max_period = | |
FailuresSrcMaxPeriod, | |
failures_source = FailuresSrc, | |
failures_dest_die = FailuresDstDie, | |
failures_dest_max_count = | |
FailuresDstMaxCount, | |
failures_dest_max_period = | |
FailuresDstMaxPeriod, | |
failures_dest = FailuresDst, | |
requests = Requests, | |
pending = Pending} = State, | |
Dispatcher) -> | |
{#pending{trans_id = TransId, | |
dest = DstPid}, | |
NewPending} = maps:take(QuorumTransId, Pending), | |
#request{% return data | |
request_type = RequestType, | |
name = Name, | |
pattern = Pattern, | |
source = SrcPid, | |
% quorum data | |
count_required = CountRequired, | |
count_total = CountTotal, | |
count_responses = CountResponses, | |
count_correct = CountCorrect, | |
responses = Responses, | |
returned = Returned} = Request = maps:get(TransId, Requests), | |
NewCountResponses = CountResponses + 1, | |
case validate(ResponseInfoF, ResponseF, | |
ResponseInfo, Response) of | |
true -> | |
NewRequests = if | |
Returned =:= false -> | |
Key = if | |
UseResponseInfo =:= true -> | |
{ResponseInfo, Response}; | |
UseResponseInfo =:= false -> | |
Response | |
end, | |
Count = case orddict:find(Key, Responses) of | |
{ok, I} -> | |
I + 1; | |
error -> | |
1 | |
end, | |
NewCountCorrect = erlang:max(Count, CountCorrect), | |
NewReturned = if | |
Count == CountRequired -> | |
cloudi_service: | |
return_nothrow(Dispatcher, RequestType, | |
Name, Pattern, | |
ResponseInfo, Response, | |
Timeout, TransId, SrcPid), | |
true; | |
(NewCountResponses == CountTotal) orelse | |
((NewCountCorrect + | |
(CountTotal - NewCountResponses)) < CountRequired) -> | |
cloudi_service: | |
return_nothrow(Dispatcher, RequestType, | |
Name, Pattern, | |
<<>>, <<>>, | |
Timeout, TransId, SrcPid), | |
true; | |
true -> | |
Returned | |
end, | |
if | |
NewCountResponses == CountTotal -> | |
maps:remove(TransId, Requests); | |
NewCountResponses < CountTotal -> | |
NewResponses = orddict:store(Key, Count, Responses), | |
maps:put(TransId, | |
Request#request{ | |
count_responses = NewCountResponses, | |
count_correct = NewCountCorrect, | |
responses = NewResponses, | |
returned = NewReturned}, | |
Requests) | |
end; | |
NewCountResponses == CountTotal -> | |
maps:remove(TransId, Requests); | |
NewCountResponses < CountTotal -> | |
maps:put(TransId, | |
Request#request{ | |
count_responses = NewCountResponses}, | |
Requests) | |
end, | |
{noreply, State#state{requests = NewRequests, | |
pending = NewPending}}; | |
false -> | |
{DeadSrc, NewFailuresSrc} = failure(FailuresSrcDie, | |
FailuresSrcMaxCount, | |
FailuresSrcMaxPeriod, | |
SrcPid, FailuresSrc), | |
{_, NewFailuresDst} = failure(FailuresDstDie, | |
FailuresDstMaxCount, | |
FailuresDstMaxPeriod, | |
DstPid, FailuresDst), | |
NewReturned = if | |
DeadSrc =:= true -> | |
true; | |
(Returned =:= false) andalso | |
((NewCountResponses == CountTotal) orelse | |
((CountCorrect + | |
(CountTotal - NewCountResponses)) < CountRequired)) -> | |
cloudi_service:return_nothrow(Dispatcher, RequestType, | |
Name, Pattern, | |
<<>>, <<>>, | |
Timeout, TransId, SrcPid), | |
true; | |
true -> | |
Returned | |
end, | |
NewRequests = if | |
NewCountResponses == CountTotal -> | |
maps:remove(TransId, Requests); | |
NewCountResponses < CountTotal -> | |
maps:put(TransId, | |
Request#request{ | |
count_responses = NewCountResponses, | |
returned = NewReturned}, | |
Requests) | |
end, | |
{noreply, State#state{failures_source = NewFailuresSrc, | |
failures_dest = NewFailuresDst, | |
requests = NewRequests, | |
pending = NewPending}} | |
end; | |
cloudi_service_handle_info(#timeout_async_active{trans_id = QuorumTransId}, | |
#state{failures_source_die = FailuresSrcDie, | |
failures_source_max_count = | |
FailuresSrcMaxCount, | |
failures_source_max_period = | |
FailuresSrcMaxPeriod, | |
failures_source = FailuresSrc, | |
failures_dest_die = FailuresDstDie, | |
failures_dest_max_count = | |
FailuresDstMaxCount, | |
failures_dest_max_period = | |
FailuresDstMaxPeriod, | |
failures_dest = FailuresDst, | |
requests = Requests, | |
pending = Pending} = State, | |
Dispatcher) -> | |
{#pending{trans_id = TransId, | |
dest = DstPid}, | |
NewPending} = maps:take(QuorumTransId, Pending), | |
#request{% return data | |
request_type = RequestType, | |
name = Name, | |
pattern = Pattern, | |
timeout = Timeout, | |
source = SrcPid, | |
% quorum data | |
count_required = CountRequired, | |
count_total = CountTotal, | |
count_responses = CountResponses, | |
count_correct = CountCorrect, | |
returned = Returned} = Request = maps:get(TransId, Requests), | |
NewCountResponses = CountResponses + 1, | |
{DeadSrc, NewFailuresSrc} = failure(FailuresSrcDie, | |
FailuresSrcMaxCount, | |
FailuresSrcMaxPeriod, | |
SrcPid, FailuresSrc), | |
{_, NewFailuresDst} = failure(FailuresDstDie, | |
FailuresDstMaxCount, | |
FailuresDstMaxPeriod, | |
DstPid, FailuresDst), | |
NewReturned = if | |
DeadSrc =:= true -> | |
true; | |
(Returned =:= false) andalso | |
((NewCountResponses == CountTotal) orelse | |
((CountCorrect + | |
(CountTotal - NewCountResponses)) < CountRequired)) -> | |
cloudi_service:return_nothrow(Dispatcher, RequestType, | |
Name, Pattern, | |
<<>>, <<>>, | |
Timeout, TransId, SrcPid), | |
true; | |
true -> | |
Returned | |
end, | |
NewRequests = if | |
NewCountResponses == CountTotal -> | |
maps:remove(TransId, Requests); | |
NewCountResponses < CountTotal -> | |
maps:put(TransId, | |
Request#request{count_responses = NewCountResponses, | |
returned = NewReturned}, | |
Requests) | |
end, | |
{noreply, State#state{failures_source = NewFailuresSrc, | |
failures_dest = NewFailuresDst, | |
requests = NewRequests, | |
pending = NewPending}}; | |
cloudi_service_handle_info({'DOWN', _MonitorRef, process, Pid, _Info}, | |
#state{failures_source_die = FailuresSrcDie, | |
failures_source = FailuresSrc, | |
failures_dest_die = FailuresDstDie, | |
failures_dest = FailuresDst} = State, | |
_Dispatcher) -> | |
NewFailuresSrc = if | |
FailuresSrcDie =:= true -> | |
maps:remove(Pid, FailuresSrc); | |
FailuresSrcDie =:= false -> | |
FailuresSrc | |
end, | |
NewFailuresDst = if | |
FailuresDstDie =:= true -> | |
maps:remove(Pid, FailuresDst); | |
FailuresDstDie =:= false -> | |
FailuresDst | |
end, | |
{noreply, State#state{failures_source = NewFailuresSrc, | |
failures_dest = NewFailuresDst}}; | |
cloudi_service_handle_info(Request, State, _Dispatcher) -> | |
{stop, cloudi_string:format("Unknown info \"~w\"", [Request]), State}. | |
cloudi_service_terminate(_Reason, _Timeout, _State) -> | |
ok. | |
%%%------------------------------------------------------------------------ | |
%%% Private functions | |
%%%------------------------------------------------------------------------ | |
mcast_send(PatternPids, Pending, QuorumName, | |
RequestInfo, Request, Timeout, Priority, TransId, Dispatcher) -> | |
mcast_send(PatternPids, 0, Pending, QuorumName, | |
RequestInfo, Request, Timeout, Priority, TransId, Dispatcher). | |
mcast_send([], CountSent, Pending, _, _, _, _, _, _, _) -> | |
{CountSent, Pending}; | |
mcast_send([{_, DstPid} = PatternPid | PatternPids], | |
CountSent, Pending, QuorumName, | |
RequestInfo, Request, Timeout, Priority, TransId, Dispatcher) -> | |
case cloudi_service:send_async_active(Dispatcher, QuorumName, | |
RequestInfo, Request, | |
Timeout, Priority, PatternPid) of | |
{ok, QuorumTransId} -> | |
mcast_send(PatternPids, CountSent + 1, | |
maps:put(QuorumTransId, | |
#pending{trans_id = TransId, | |
dest = DstPid}, | |
Pending), | |
QuorumName, | |
RequestInfo, Request, | |
Timeout, Priority, TransId, Dispatcher); | |
{error, timeout} -> | |
mcast_send(PatternPids, CountSent, | |
Pending, QuorumName, | |
RequestInfo, Request, | |
Timeout, Priority, TransId, Dispatcher) | |
end. | |
mcast(PatternPids, QuorumName, | |
RequestType, Name, Pattern, RequestInfo, Request, | |
Timeout, Priority, TransId, SrcPid, CountRequired, | |
#state{requests = Requests, | |
pending = Pending} = State, Dispatcher) -> | |
{CountSent, | |
NewPending} = mcast_send(PatternPids, Pending, QuorumName, | |
RequestInfo, Request, | |
Timeout, Priority, TransId, Dispatcher), | |
if | |
CountSent >= CountRequired -> | |
NewRequests = maps:put(TransId, | |
#request{request_type = RequestType, | |
name = Name, | |
pattern = Pattern, | |
timeout = Timeout, | |
source = SrcPid, | |
count_required = CountRequired, | |
count_total = CountSent, | |
returned = false}, | |
Requests), | |
{noreply, State#state{requests = NewRequests, | |
pending = NewPending}}; | |
true -> | |
NewRequests = maps:put(TransId, | |
#request{request_type = RequestType, | |
name = Name, | |
pattern = Pattern, | |
timeout = Timeout, | |
source = SrcPid, | |
count_required = CountRequired, | |
count_total = CountSent, | |
returned = true}, | |
Requests), | |
{reply, <<>>, State#state{requests = NewRequests, | |
pending = NewPending}} | |
end. | |
validate_f_return(Value) when is_boolean(Value) -> | |
Value. | |
validate(undefined, undefined, _, _) -> | |
true; | |
validate(undefined, RF, RInfo, R) -> | |
validate_f_return(RF(RInfo, R)); | |
validate(RInfoF, undefined, RInfo, _) -> | |
validate_f_return(RInfoF(RInfo)); | |
validate(RInfoF, RF, RInfo, R) -> | |
validate_f_return(RInfoF(RInfo)) andalso validate_f_return(RF(RInfo, R)). | |
request_failed(SrcPid, | |
#state{failures_source_die = FailuresSrcDie, | |
failures_source_max_count = FailuresSrcMaxCount, | |
failures_source_max_period = FailuresSrcMaxPeriod, | |
failures_source = FailuresSrc} = State) -> | |
{DeadSrc, NewFailuresSrc} = failure(FailuresSrcDie, | |
FailuresSrcMaxCount, | |
FailuresSrcMaxPeriod, | |
SrcPid, FailuresSrc), | |
if | |
DeadSrc =:= true -> | |
{noreply, | |
State#state{failures_source = NewFailuresSrc}}; | |
DeadSrc =:= false -> | |
{reply, <<>>, | |
State#state{failures_source = NewFailuresSrc}} | |
end. | |
failure(false, _, _, _, Failures) -> | |
{false, Failures}; | |
failure(true, MaxCount, MaxPeriod, Pid, Failures) -> | |
case erlang:is_process_alive(Pid) of | |
true -> | |
SecondsNow = cloudi_timestamp:seconds_monotonic(), | |
case maps:find(Pid, Failures) of | |
{ok, FailureList} -> | |
failure_check(SecondsNow, FailureList, | |
MaxCount, MaxPeriod, Pid, Failures); | |
error -> | |
erlang:monitor(process, Pid), | |
failure_check(SecondsNow, [], | |
MaxCount, MaxPeriod, Pid, Failures) | |
end; | |
false -> | |
{true, Failures} | |
end. | |
failure_store(FailureList, FailureCount, MaxCount, Pid, Failures) -> | |
NewFailures = maps:put(Pid, FailureList, Failures), | |
if | |
FailureCount == MaxCount -> | |
failure_kill(Pid), | |
{true, NewFailures}; | |
true -> | |
{false, NewFailures} | |
end. | |
failure_check(SecondsNow, FailureList, MaxCount, infinity, Pid, Failures) -> | |
NewFailureCount = erlang:length(FailureList), | |
failure_store([SecondsNow | FailureList], NewFailureCount + 1, | |
MaxCount, Pid, Failures); | |
failure_check(SecondsNow, FailureList, MaxCount, MaxPeriod, Pid, Failures) -> | |
{NewFailureCount, | |
NewFailureList} = cloudi_timestamp:seconds_filter_monotonic(FailureList, | |
SecondsNow, | |
MaxPeriod), | |
failure_store([SecondsNow | NewFailureList], NewFailureCount + 1, | |
MaxCount, Pid, Failures). | |
failure_kill(Pid) -> | |
erlang:exit(Pid, cloudi_service_quorum). |