Merge d439056 into e67b078

apps/emqx/src/emqx_schema.erl

@@ -93,7 +93,6 @@
     validate_heap_size/1,
     validate_packet_size/1,
     user_lookup_fun_tr/2,
-    validate_alarm_actions/1,
     validate_keepalive_multiplier/1,
     non_empty_string/1,
     validations/0,
@@ -1617,10 +1616,9 @@ fields("alarm") ->
     [
         {"actions",
             sc(
-                hoconsc:array(atom()),
+                hoconsc:array(hoconsc:enum([log, publish])),
                 #{
                     default => [log, publish],
-                    validator => fun ?MODULE:validate_alarm_actions/1,
                     example => [log, publish],
                     desc => ?DESC(alarm_actions)
                 }
@@ -2777,15 +2775,6 @@ validate_keepalive_multiplier(Multiplier) when
 validate_keepalive_multiplier(_Multiplier) ->
     {error, #{reason => keepalive_multiplier_out_of_range, min => 1, max => 65535}}.
 
-validate_alarm_actions(Actions) ->
-    UnSupported = lists:filter(
-        fun(Action) -> Action =/= log andalso Action =/= publish end, Actions
-    ),
-    case UnSupported of
-        [] -> ok;
-        Error -> {error, Error}
-    end.
-
 validate_tcp_keepalive(Value) ->
     case iolist_to_binary(Value) of
         <<"none">> ->

apps/emqx_auth/include/emqx_authn.hrl

@@ -21,8 +21,6 @@
 
 -define(AUTHN, emqx_authn_chains).
 
--define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}").
-
 %% has to be the same as the root field name defined in emqx_schema
 -define(CONF_NS, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME).
 -define(CONF_NS_ATOM, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
@@ -32,4 +30,16 @@
 
 -define(AUTHN_RESOURCE_GROUP, <<"emqx_authn">>).
 
+%% VAR_NS_CLIENT_ATTRS is added here because it can be initialized before authn.
+%% NOTE: authn return may add more to (or even overwrite) client_attrs.
+-define(AUTHN_DEFAULT_ALLOWED_VARS, [
+    ?VAR_USERNAME,
+    ?VAR_CLIENTID,
+    ?VAR_PASSWORD,
+    ?VAR_PEERHOST,
+    ?VAR_CERT_SUBJECT,
+    ?VAR_CERT_CN_NAME,
+    ?VAR_NS_CLIENT_ATTRS
+]).
+
 -endif.

apps/emqx_auth/include/emqx_authz.hrl

@@ -38,8 +38,6 @@
 -define(ROOT_KEY, [authorization]).
 -define(CONF_KEY_PATH, [authorization, sources]).
 
--define(RE_PLACEHOLDER, "\\$\\{[a-z0-9_]+\\}").
-
 %% has to be the same as the root field name defined in emqx_schema
 -define(CONF_NS, ?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME).
 -define(CONF_NS_ATOM, ?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM).

apps/emqx_auth/src/emqx_auth_utils.erl

@@ -16,15 +16,221 @@
 
 -module(emqx_auth_utils).
 
-%% TODO
-%% Move more identical authn and authz helpers here
+-include_lib("emqx/include/emqx_placeholder.hrl").
+-include_lib("snabbkaffe/include/trace.hrl").
 
+%% Template parsing/rendering
+-export([
+    parse_deep/2,
+    parse_str/2,
+    parse_sql/3,
+    render_deep_for_json/2,
+    render_deep_for_url/2,
+    render_deep_for_raw/2,
+    render_str/2,
+    render_urlencoded_str/2,
+    render_sql_params/2
+]).
+
+%% URL parsing
 -export([parse_url/1]).
 
+%% HTTP request/response helpers
+-export([generate_request/2]).
+
+-define(DEFAULT_HTTP_REQUEST_CONTENT_TYPE, <<"application/json">>).
+
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
 
+%%--------------------------------------------------------------------
+%% Template parsing/rendering
+
+parse_deep(Template, AllowedVars) ->
+    Result = emqx_template:parse_deep(Template),
+    handle_disallowed_placeholders(Result, AllowedVars, {deep, Template}).
+
+parse_str(Template, AllowedVars) ->
+    Result = emqx_template:parse(Template),
+    handle_disallowed_placeholders(Result, AllowedVars, {string, Template}).
+
+parse_sql(Template, ReplaceWith, AllowedVars) ->
+    {Statement, Result} = emqx_template_sql:parse_prepstmt(
+        Template,
+        #{parameters => ReplaceWith, strip_double_quote => true}
+    ),
+    {Statement, handle_disallowed_placeholders(Result, AllowedVars, {string, Template})}.
+
+handle_disallowed_placeholders(Template, AllowedVars, Source) ->
+    case emqx_template:validate(AllowedVars, Template) of
+        ok ->
+            Template;
+        {error, Disallowed} ->
+            ?tp(warning, "auth_template_invalid", #{
+                template => Source,
+                reason => Disallowed,
+                allowed => #{placeholders => AllowedVars},
+                notice =>
+                    "Disallowed placeholders will be rendered as is."
+                    " However, consider using `${$}` escaping for literal `$` where"
+                    " needed to avoid unexpected results."
+            }),
+            Result = prerender_disallowed_placeholders(Template, AllowedVars),
+            case Source of
+                {string, _} ->
+                    emqx_template:parse(Result);
+                {deep, _} ->
+                    emqx_template:parse_deep(Result)
+            end
+    end.
+
+prerender_disallowed_placeholders(Template, AllowedVars) ->
+    {Result, _} = emqx_template:render(Template, #{}, #{
+        var_trans => fun(Name, _) ->
+            % NOTE
+            % Rendering disallowed placeholders in escaped form, which will then
+            % parse as a literal string.
+            case lists:member(Name, AllowedVars) of
+                true -> "${" ++ Name ++ "}";
+                false -> "${$}{" ++ Name ++ "}"
+            end
+        end
+    }),
+    Result.
+
+render_deep_for_json(Template, Credential) ->
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {Term, _Errors} = emqx_template:render(
+        Template,
+        rename_client_info_vars(Credential),
+        #{var_trans => fun to_string_for_json/2}
+    ),
+    Term.
+
+render_deep_for_raw(Template, Credential) ->
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {Term, _Errors} = emqx_template:render(
+        Template,
+        rename_client_info_vars(Credential),
+        #{var_trans => fun to_string_for_raw/2}
+    ),
+    Term.
+
+render_deep_for_url(Template, Credential) ->
+    render_deep_for_raw(Template, Credential).
+
+render_str(Template, Credential) ->
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {String, _Errors} = emqx_template:render(
+        Template,
+        rename_client_info_vars(Credential),
+        #{var_trans => fun to_string/2}
+    ),
+    unicode:characters_to_binary(String).
+
+render_urlencoded_str(Template, Credential) ->
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {String, _Errors} = emqx_template:render(
+        Template,
+        rename_client_info_vars(Credential),
+        #{var_trans => fun to_urlencoded_string/2}
+    ),
+    unicode:characters_to_binary(String).
+
+render_sql_params(ParamList, Credential) ->
+    % NOTE
+    % Ignoring errors here, undefined bindings will be replaced with empty string.
+    {Row, _Errors} = emqx_template:render(
+        ParamList,
+        rename_client_info_vars(Credential),
+        #{var_trans => fun to_sql_value/2}
+    ),
+    Row.
+
+to_urlencoded_string(Name, Value) ->
+    case uri_string:compose_query([{<<"q">>, to_string(Name, Value)}]) of
+        <<"q=", EncodedBin/binary>> ->
+            EncodedBin;
+        "q=" ++ EncodedStr ->
+            list_to_binary(EncodedStr)
+    end.
+
+to_string(Name, Value) ->
+    emqx_template:to_string(render_var(Name, Value)).
+
+%% This converter is to generate data structure possibly with non-utf8 strings.
+
+to_string_for_raw(Name, Value) ->
+    to_string_for_raw(render_var(Name, Value)).
+
+to_string_for_raw(Value) when is_binary(Value) ->
+    Value;
+to_string_for_raw(Value) when is_list(Value) ->
+    unicode:characters_to_binary(Value);
+to_string_for_raw(Value) ->
+    emqx_template:to_string(Value).
+
+%% JSON strings are sequences of unicode characters, not bytes.
+%% So we force all rendered data to be unicode.
+
+to_string_for_json(Name, Value) ->
+    to_unicode_string(Name, render_var(Name, Value)).
+
+to_unicode_string(Name, Value) when is_list(Value) orelse is_binary(Value) ->
+    try unicode:characters_to_binary(Value) of
+        Encoded when is_binary(Encoded) ->
+            Encoded;
+        _ ->
+            error({encode_error, {non_unicode_data, Name}})
+    catch
+        error:badarg ->
+            error({encode_error, {non_unicode_data, Name}})
+    end;
+to_unicode_string(_Name, Value) ->
+    emqx_template:to_string(Value).
+
+to_sql_value(Name, Value) ->
+    emqx_utils_sql:to_sql_value(render_var(Name, Value)).
+
+render_var(_, undefined) ->
+    % NOTE
+    % Any allowed but undefined binding will be replaced with empty string, even when
+    % rendering SQL values.
+    <<>>;
+render_var(?VAR_PEERHOST, Value) ->
+    inet:ntoa(Value);
+render_var(?VAR_PASSWORD, Value) ->
+    iolist_to_binary(Value);
+render_var(_Name, Value) ->
+    Value.
+
+rename_client_info_vars(ClientInfo) ->
+    Renames = [
+        {cn, cert_common_name},
+        {dn, cert_subject},
+        {protocol, proto_name}
+    ],
+    lists:foldl(
+        fun({Old, New}, Acc) ->
+            case Acc of
+                #{Old := Value} ->
+                    maps:put(New, Value, maps:remove(Old, Acc));
+                _ ->
+                    Acc
+            end
+        end,
+        ClientInfo,
+        Renames
+    ).
+
+%%--------------------------------------------------------------------
+%% URL parsing
+
 -spec parse_url(binary()) ->
     {_Base :: emqx_utils_uri:request_base(), _Path :: binary(), _Query :: binary()}.
 parse_url(Url) ->
@@ -48,6 +254,59 @@ parse_url(Url) ->
             end
     end.
 
+%%--------------------------------------------------------------------
+%% HTTP request/response helpers
+
+generate_request(
+    #{
+        method := Method,
+        headers := Headers,
+        base_path_template := BasePathTemplate,
+        base_query_template := BaseQueryTemplate,
+        body_template := BodyTemplate
+    },
+    Values
+) ->
+    Path = emqx_auth_utils:render_urlencoded_str(BasePathTemplate, Values),
+    Query = emqx_auth_utils:render_deep_for_url(BaseQueryTemplate, Values),
+    case Method of
+        get ->
+            Body = emqx_auth_utils:render_deep_for_url(BodyTemplate, Values),
+            NPath = append_query(Path, Query, Body),
+            {ok, {NPath, Headers}};
+        _ ->
+            try
+                ContentType = post_request_content_type(Headers),
+                Body = serialize_body(ContentType, BodyTemplate, Values),
+                NPathQuery = append_query(Path, Query),
+                {ok, {NPathQuery, Headers, Body}}
+            catch
+                error:{encode_error, _} = Reason ->
+                    {error, Reason}
+            end
+    end.
+
+post_request_content_type(Headers) ->
+    proplists:get_value(<<"content-type">>, Headers, ?DEFAULT_HTTP_REQUEST_CONTENT_TYPE).
+
+append_query(Path, []) ->
+    Path;
+append_query(Path, Query) ->
+    [Path, $?, uri_string:compose_query(Query)].
+append_query(Path, Query, Body) ->
+    append_query(Path, Query ++ maps:to_list(Body)).
+
+serialize_body(<<"application/json">>, BodyTemplate, ClientInfo) ->
+    Body = emqx_auth_utils:render_deep_for_json(BodyTemplate, ClientInfo),
+    emqx_utils_json:encode(Body);
+serialize_body(<<"application/x-www-form-urlencoded">>, BodyTemplate, ClientInfo) ->
+    Body = emqx_auth_utils:render_deep_for_url(BodyTemplate, ClientInfo),
+    uri_string:compose_query(maps:to_list(Body));
+serialize_body(undefined, _BodyTemplate, _ClientInfo) ->
+    throw(missing_content_type_header);
+serialize_body(ContentType, _BodyTemplate, _ClientInfo) ->
+    throw({unknown_content_type_header_value, ContentType}).
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 

apps/emqx_auth/src/emqx_authn/emqx_authn_chains.erl

@@ -678,16 +678,28 @@ do_authenticate(
             {stop, Result}
     catch
         Class:Reason:Stacktrace ->
-            ?TRACE_AUTHN(warning, "authenticator_error", #{
-                exception => Class,
-                reason => Reason,
-                stacktrace => Stacktrace,
-                authenticator => ID
-            }),
+            ?TRACE_AUTHN(
+                warning,
+                "authenticator_error",
+                maybe_add_stacktrace(
+                    Class,
+                    #{
+                        exception => Class,
+                        reason => Reason,
+                        authenticator => ID
+                    },
+                    Stacktrace
+                )
+            ),
             emqx_metrics_worker:inc(authn_metrics, MetricsID, nomatch),
             do_authenticate(ChainName, More, Credential)
     end.
 
+maybe_add_stacktrace('throw', Data, _Stacktrace) ->
+    Data;
+maybe_add_stacktrace(_, Data, Stacktrace) ->
+    Data#{stacktrace => Stacktrace}.
+
 authenticate_with_provider(#authenticator{id = ID, provider = Provider, state = State}, Credential) ->
     AuthnResult = Provider:authenticate(Credential, State),
     ?TRACE_AUTHN("authenticator_result", #{