Skip to content

Commit

Permalink
Initial logger support (vernemq#2230)
Browse files Browse the repository at this point in the history
* Initial logger support

* Apply suggestions from code review

Address code review

Co-authored-by: Dairon M. <dairon.medina@gmail.com>

* Address review comments (log show -> log show config)

---------

Co-authored-by: mths1 <mths1>
Co-authored-by: Dairon M. <dairon.medina@gmail.com>
  • Loading branch information
2 people authored and mths1 committed Feb 9, 2024
1 parent 47d7914 commit 11e8525
Show file tree
Hide file tree
Showing 103 changed files with 914 additions and 468 deletions.
1 change: 0 additions & 1 deletion apps/vmq_bridge/rebar.config
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
{erl_opts, [
{parse_transform, lager_transform},
warnings_as_errors,
debug_info
]}.
9 changes: 5 additions & 4 deletions apps/vmq_bridge/src/vmq_bridge.erl
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
%% limitations under the License.

-module(vmq_bridge).
-include_lib("kernel/include/logger.hrl").

-behaviour(gen_mqtt_client).

Expand Down Expand Up @@ -91,7 +92,7 @@ on_connect({coord, CoordinatorPid} = State) ->
{ok, State}.

on_connect_error(Reason, State) ->
lager:error("connection failed due to ~p", [Reason]),
?LOG_ERROR("connection failed due to ~p", [Reason]),
{ok, State}.

on_disconnect(State) ->
Expand All @@ -104,9 +105,9 @@ on_subscribe(Topics, {coord, CoordPid} = State) ->
],
case FailedTopics of
[] ->
lager:info("Bridge Pid ~p is subscribing to Topics: ~p~n", [CoordPid, Topics]);
?LOG_INFO("Bridge Pid ~p is subscribing to Topics: ~p~n", [CoordPid, Topics]);
_ ->
lager:warning(
?LOG_WARNING(
"Bridge Pid ~p had subscription failure codes in SUBACK for topics ~p~n", [
CoordPid, FailedTopics
]
Expand Down Expand Up @@ -207,7 +208,7 @@ handle_info(
subscribe_fun = SubscribeFun
} = State
) ->
lager:info("Bridge ~s connected to ~s:~p.~n", [Name, Host, Port]),
?LOG_INFO("Bridge ~s connected to ~s:~p.~n", [Name, Host, Port]),
Topics = proplists:get_value(topics, Opts),
Subscriptions = bridge_subscribe(remote, Pid, Topics, SubscribeFun, []),
{noreply, State#state{subs_remote = Subscriptions}};
Expand Down
20 changes: 20 additions & 0 deletions apps/vmq_commons/src/vmq_log.erl
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%% http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.

-module(vmq_log).
-include_lib("kernel/include/logger.hrl").

-export([set_loglevel/2]).

set_loglevel(Logger, Level) ->
?LOG_INFO("Changed log level for Logger ~p to ~p", [Logger, Level]),
logger:update_handler_config(list_to_atom(Logger), level, list_to_atom(Level)).
216 changes: 216 additions & 0 deletions apps/vmq_commons/src/vmq_log_json_format.erl
Original file line number Diff line number Diff line change
@@ -0,0 +1,216 @@
%%
%% Based on rabbit_logger_json_fmt and rabbit_logger_fmt_helpers.erl
%%
%% This Source Code Form is subject to the terms of the Mozilla Public
%% License, v. 2.0. If a copy of the MPL was not distributed with this
%% file, You can obtain one at https://mozilla.org/MPL/2.0/.
%%
%% Copyright (c) 2021 VMware, Inc. or its affiliates. All rights reserved.
%%

-module(vmq_log_json_format).

-export([format/2]).
-compile(nowarn_unused_function).
-compile(nowarn_unused_vars).

format(
#{
msg := Msg,
level := Level,
meta := Meta
},
Config
) ->
FormattedLevel = unicode:characters_to_binary(atom_to_list(Level)),
FormattedMeta = format_meta(Meta, Config),
%% We need to call `unicode:characters_to_binary()' here and several other
%% places because JSON encoder will format a string as a list of
%% integers (we don't blame it for that, it makes sense).
FormattedMsg = unicode:characters_to_binary(format_msg(Msg, Meta, Config)),
InitialDoc0 = FormattedMeta#{
level => FormattedLevel,
msg => FormattedMsg
},
InitialDoc =
case level_to_verbosity(Level, Config) of
undefined -> InitialDoc0;
Verbosity -> InitialDoc0#{verbosity => Verbosity}
end,
DocAfterMapping = apply_mapping_and_ordering(InitialDoc, Config),
Json = vmq_json:encode(DocAfterMapping),
[Json, $\n].

level_to_verbosity(Level, #{verbosity_map := Mapping}) ->
case maps:is_key(Level, Mapping) of
true -> maps:get(Level, Mapping);
false -> undefined
end;
level_to_verbosity(_, _) ->
undefined.

format_meta(Meta, _Config) ->
maps:fold(
fun
(time, Timestamp, Acc) ->
FormattedTime0 = Timestamp,
FormattedTime1 =
case is_number(FormattedTime0) of
true ->
FormattedTime0;
false ->
unicode:characters_to_binary(
FormattedTime0
)
end,
Acc#{time => FormattedTime1};
(domain = Key, Components, Acc) ->
Term = unicode:characters_to_binary(
string:join(
[atom_to_list(Cmp) || Cmp <- Components],
"."
)
),
Acc#{Key => Term};
(Key, Value, Acc) ->
case convert_to_types_accepted_by_json_encoder(Value) of
false -> Acc;
Term -> Acc#{Key => Term}
end
end,
#{},
Meta
).

convert_to_types_accepted_by_json_encoder(Term) when is_map(Term) ->
maps:map(
fun(_, Value) -> convert_to_types_accepted_by_json_encoder(Value) end,
Term
);
convert_to_types_accepted_by_json_encoder(Term) when is_list(Term) ->
case io_lib:deep_char_list(Term) of
true ->
unicode:characters_to_binary(Term);
false ->
[convert_to_types_accepted_by_json_encoder(E) || E <- Term]
end;
convert_to_types_accepted_by_json_encoder(Term) when is_tuple(Term) ->
convert_to_types_accepted_by_json_encoder(erlang:tuple_to_list(Term));
convert_to_types_accepted_by_json_encoder(Term) when is_function(Term) ->
String = erlang:fun_to_list(Term),
unicode:characters_to_binary(String);
convert_to_types_accepted_by_json_encoder(Term) when is_pid(Term) ->
String = erlang:pid_to_list(Term),
unicode:characters_to_binary(String);
convert_to_types_accepted_by_json_encoder(Term) when is_port(Term) ->
String = erlang:port_to_list(Term),
unicode:characters_to_binary(String);
convert_to_types_accepted_by_json_encoder(Term) when is_reference(Term) ->
String = erlang:ref_to_list(Term),
unicode:characters_to_binary(String);
convert_to_types_accepted_by_json_encoder(Term) ->
Term.

apply_mapping_and_ordering(Doc, #{field_map := Mapping}) ->
apply_mapping_and_ordering(Mapping, Doc, []);
apply_mapping_and_ordering(Doc, _) ->
maps:to_list(Doc).

apply_mapping_and_ordering([{'$REST', false} | Rest], _, Result) ->
apply_mapping_and_ordering(Rest, #{}, Result);
apply_mapping_and_ordering([{Old, false} | Rest], Doc, Result) when
is_atom(Old)
->
Doc1 = maps:remove(Old, Doc),
apply_mapping_and_ordering(Rest, Doc1, Result);
apply_mapping_and_ordering([{Old, New} | Rest], Doc, Result) when
is_atom(Old) andalso is_atom(New)
->
case maps:is_key(Old, Doc) of
true ->
Value = maps:get(Old, Doc),
Doc1 = maps:remove(Old, Doc),
Result1 = [{New, Value} | Result],
apply_mapping_and_ordering(Rest, Doc1, Result1);
false ->
apply_mapping_and_ordering(Rest, Doc, Result)
end;
apply_mapping_and_ordering([], Doc, Result) ->
lists:reverse(Result) ++ maps:to_list(Doc).

format_msg(Msg, Meta, #{single_line := true} = Config) ->
FormattedMsg = format_msg1(Msg, Meta, Config),
%% The following behavior is the same as the one in the official
%% `logger_formatter'; the code is taken from:
%% https://github.com/erlang/otp/blob/c5ed910098e9c2787e2c3f9f462c84322064e00d/lib/kernel/src/logger_formatter.erl
FormattedMsg1 = string:strip(FormattedMsg, both),
re:replace(
FormattedMsg1,
",?\r?\n\s*",
", ",
[{return, list}, global, unicode]
);
format_msg(Msg, Meta, Config) ->
format_msg1(Msg, Meta, Config).

format_msg1({string, Chardata}, Meta, Config) ->
format_msg1({"~ts", [Chardata]}, Meta, Config);
format_msg1({report, Report}, Meta, Config) ->
FormattedReport = format_report(Report, Meta, Config),
format_msg1(FormattedReport, Meta, Config);
format_msg1({Format, Args}, _, _) ->
io_lib:format(Format, Args).

format_report(
#{label := {application_controller, _}} = Report, Meta, Config
) ->
format_application_progress(Report, Meta, Config);
format_report(
#{label := {supervisor, progress}} = Report, Meta, Config
) ->
format_supervisor_progress(Report, Meta, Config);
format_report(
Report, #{report_cb := Cb} = Meta, Config
) ->
try
case erlang:fun_info(Cb, arity) of
{arity, 1} -> Cb(Report);
{arity, 2} -> {"~ts", [Cb(Report, #{})]}
end
catch
_:_:_ ->
format_report(Report, maps:remove(report_cb, Meta), Config)
end;
format_report(Report, _, _) ->
logger:format_report(Report).

format_application_progress(
#{
label := {_, progress},
report := InternalReport
},
_,
_
) ->
Application = proplists:get_value(application, InternalReport),
StartedAt = proplists:get_value(started_at, InternalReport),
{"Application ~w started on ~0p", [Application, StartedAt]};
format_application_progress(
#{
label := {_, exit},
report := InternalReport
},
_,
_
) ->
Application = proplists:get_value(application, InternalReport),
Exited = proplists:get_value(exited, InternalReport),
{"Application ~w exited with reason: ~0p", [Application, Exited]}.

format_supervisor_progress(#{report := InternalReport}, _, _) ->
Supervisor = proplists:get_value(supervisor, InternalReport),
Started = proplists:get_value(started, InternalReport),
Id = proplists:get_value(id, Started),
Pid = proplists:get_value(pid, Started),
Mfa = proplists:get_value(mfargs, Started),
{"Supervisor ~w: child ~w started (~w): ~0p", [Supervisor, Id, Pid, Mfa]}.
3 changes: 1 addition & 2 deletions apps/vmq_diversity/rebar.config
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
{erl_opts, [
{parse_transform, lager_transform},
warnings_as_errors,
debug_info
]}.
Expand All @@ -17,5 +16,5 @@
%%Eonblast hasn't merged the Erlang 18 related PR from djustinek
%%{emysql, {git, "git://github.com/Eonblast/Emysql.git", {tag, "v0.4.1"}}},
{mongodb, {git, "https://github.com/comtihon/mongodb-erlang.git", {ref, "713e8bd"}}},
{mcd, {git, "https://github.com/EchoTeam/mcd.git", {ref, "b5b4a32"}}}
{mcd, {git, "https://github.com/mths1/mcd.git", {branch, "master"}}}
]}.
1 change: 0 additions & 1 deletion apps/vmq_diversity/src/vmq_diversity.app.src
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
public_key,
ssl,
eldap,
lager,
clique,
poolboy,
luerl,
Expand Down
9 changes: 5 additions & 4 deletions apps/vmq_diversity/src/vmq_diversity_app.erl
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
%%%-------------------------------------------------------------------

-module('vmq_diversity_app').
-include_lib("kernel/include/logger.hrl").

-behaviour(application).

Expand Down Expand Up @@ -66,11 +67,11 @@ start(_StartType, _StartArgs) ->
filelib:wildcard(DataDir ++ "/*.lua")
);
false ->
lager:warning("can't initialize Lua scripts using ~p", [DataDir])
?LOG_WARNING("can't initialize Lua scripts using ~p", [DataDir])
end,
lists:foreach(
fun({_Name, Script}) ->
lager:info("enable script for ~p", [Script]),
?LOG_INFO("enable script for ~p", [Script]),
load_script(Script)
end,
application:get_env(vmq_diversity, user_scripts, [])
Expand All @@ -80,7 +81,7 @@ start(_StartType, _StartArgs) ->
case proplists:get_value(enabled, AuthScriptConfig, false) of
true ->
Script = proplists:get_value(file, AuthScriptConfig),
lager:info("enable auth script for ~p ~p", [M, Script]),
?LOG_INFO("enable auth script for ~p ~p", [M, Script]),
load_script(Script);
false ->
ignore
Expand All @@ -106,5 +107,5 @@ load_script(Script) ->
{ok, _Pid} ->
ok;
{error, Reason} ->
lager:error("could not load script ~p due to ~p", [Script, Reason])
?LOG_ERROR("could not load script ~p due to ~p", [Script, Reason])
end.
7 changes: 4 additions & 3 deletions apps/vmq_diversity/src/vmq_diversity_cache.erl
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
%%%-------------------------------------------------------------------

-module(vmq_diversity_cache).
-include_lib("kernel/include/logger.hrl").
-include_lib("luerl/include/luerl.hrl").

-dialyzer(no_undefined_callbacks).
Expand Down Expand Up @@ -390,15 +391,15 @@ validate_acl(MP, User, ClientId, Rec0, [{<<"pattern">>, Pattern} | Rest]) when i
{ok, Words} when Type == subscribe ->
Rec0#subscribe_acl{pattern = subst(MP, User, ClientId, Words, [])};
{error, Reason} ->
lager:error(
?LOG_ERROR(
"can't validate ACL topic ~p for client ~p due to ~p",
[Pattern, ClientId, Reason]
),
Rec0
end,
validate_acl(MP, User, ClientId, Rec1, Rest);
validate_acl(MP, User, ClientId, Rec, [UnknownProp | Rest]) ->
lager:warning("unknown property ~p for ACL ~p", [UnknownProp, Rec]),
?LOG_WARNING("unknown property ~p for ACL ~p", [UnknownProp, Rec]),
validate_acl(MP, User, ClientId, Rec, Rest);
validate_acl(_, _, _, Rec, []) ->
Rec.
Expand All @@ -418,7 +419,7 @@ validate_modifiers(Type, Mods0) ->
end,
case Ret of
error ->
lager:error("can't validate modifiers ~p for ~p ACL", [Type, Mods0]),
?LOG_ERROR("can't validate modifiers ~p for ~p ACL", [Type, Mods0]),
undefined;
_ ->
Ret
Expand Down
3 changes: 2 additions & 1 deletion apps/vmq_diversity/src/vmq_diversity_ets.erl
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
%% limitations under the License.

-module(vmq_diversity_ets).
-include_lib("kernel/include/logger.hrl").
-include_lib("luerl/include/luerl.hrl").

-export([install/1]).
Expand Down Expand Up @@ -103,6 +104,6 @@ table_id(BTableName, As, St) ->
ATableName -> ATableName
catch
_:_ ->
lager:error("unknown pool ~p", [BTableName]),
?LOG_ERROR("unknown pool ~p", [BTableName]),
badarg_error(unknown_pool, As, St)
end.

0 comments on commit 11e8525

Please sign in to comment.