Permalink
Browse files

Adding an option for quick deduplicated delivery

When logging deduplicated messages, the error handlers only
learn about the existance of an error message after they have
happened. In some cases, the time value could be set high, and
this would create latencies in fault detection.

This commit adds an option so that the first message of a group
(seen as too different to be merged in with another message) is
instantly sent to the error logger, while the following ones get
accumulated.

This will effectively change the behaviour from receiving:
  "error X (N times)"
to:
  "error X" + "error X (N-1 times)"
  • Loading branch information...
1 parent b00dc78 commit fda26dbbcd4734593ce5c700fa1e1ea12480cae7 @ferd committed Oct 30, 2012
Showing with 281 additions and 40 deletions.
  1. +8 −1 src/lager.app.src
  2. +5 −0 src/lager_app.erl
  3. +16 −1 src/lager_deduper.erl
  4. +252 −38 test/lager_test_backend_dedup.erl
View
@@ -48,6 +48,13 @@
%% messages are identical but this limit is set to 5, the log handler
%% will only receive 5 of them and show (5 times+) instead of (N times).
%% a value of 0 or 'undefined' disables limits.
- {duplicate_limit, undefined}
+ {duplicate_limit, undefined},
+ %% Duplicate detection, by default, will eat up all the messages in a given
+ %% time period before dumping them. If the waiting time is set high (say a
+ %% minute or so), it means reaction time is much higher. Setting
+ %% 'duplicate_quick_notification' to 'true' makes the first message of a
+ %% given error go through, while accumulating the later ones to make sure
+ %% you get both quick reporting and deduplication of messages.
+ {duplicate_quick_notification, false}
]}
]}.
View
@@ -67,6 +67,11 @@ start(_StartType, _StartArgs) ->
_ -> undefined
end,
lager_mochiglobal:put(duplicate_limit, DupLimit),
+ DupQuick = case application:get_env(lager, duplicate_quick_notification) of
+ {ok, QuickDup} -> QuickDup;
+ _ -> false
+ end,
+ lager_mochiglobal:put(duplicate_quick_notification, DupQuick),
SavedHandlers = case application:get_env(lager, error_logger_redirect) of
{ok, false} ->
View
@@ -97,6 +97,10 @@ handle_call(dump, _From, S=#state{timer=Ref}) ->
{reply, ok, NewState}.
handle_cast({set, Key, Val}, S=#state{db=DB}) ->
+ case quick() of
+ true -> safe_notify(Val);
+ false -> ok
+ end,
{noreply, S#state{db=store(Key, Val, DB)}}.
handle_info({timeout, _Ref, dump}, S=#state{db=DB}) ->
@@ -112,6 +116,7 @@ terminate(_, _) -> ok.
delay() -> lager_mochiglobal:get(duplicate_dump, ?DEFAULT_TIMEOUT).
threshold() -> lager_mochiglobal:get(duplicate_threshold, 1).
limit() -> lager_mochiglobal:get(duplicate_limit, undefined).
+quick() -> lager_mochiglobal:get(duplicate_quick_notification, false).
empty() -> ets:new(?TABLE, [protected,named_table]).
@@ -128,7 +133,13 @@ increment(Key, Tab) ->
store(Key, Val, Tab) ->
case ets:update_element(Tab, Key, {3,Val}) of
- false -> ets:insert(Tab, {Key, 1, Val});
+ false ->
+ case quick() of
+ false ->
+ ets:insert(Tab, {Key, 1, Val});
+ true ->
+ ets:insert(Tab, {Key, 0, Val})
+ end;
true -> ok
end,
Tab.
@@ -157,6 +168,10 @@ dump(Tab, Current) ->
case ets:lookup(Tab, Current) of
[{_,_,undefined}] -> % may occur between hash set and log
dump(Tab, ets:next(Tab, Current));
+ [{Key, 0, _Log}] -> % handled with quick notification, discard
+ Next = ets:next(Tab, Current),
+ ets:delete(Tab, Key),
+ dump(Tab, Next);
[{Key, 1, Log = {log, _Lvl, _Ts, _Msg}}] ->
safe_notify(Log),
Next = ets:next(Tab, Current),
Oops, something went wrong.

0 comments on commit fda26db

Please sign in to comment.