Merge pull request #11634 from savonarola/0919-auth-refactor

Auth refactor

.github/CODEOWNERS

@@ -3,9 +3,9 @@
 
 ## apps
 /apps/emqx/                @emqx/emqx-review-board @lafirest
-/apps/emqx_authn/          @emqx/emqx-review-board @JimMoen @savonarola
-/apps/emqx_authz/          @emqx/emqx-review-board @JimMoen @savonarola
 /apps/emqx_connector/      @emqx/emqx-review-board
+/apps/emqx_auth/           @emqx/emqx-review-board @JimMoen @savonarola
+/apps/emqx_connector/      @emqx/emqx-review-board @JimMoen
 /apps/emqx_dashboard/      @emqx/emqx-review-board @JimMoen @lafirest
 /apps/emqx_dashboard_rbac/ @emqx/emqx-review-board @lafirest
 /apps/emqx_dashboard_sso/  @emqx/emqx-review-board @JimMoen @lafirest

.github/workflows/run_jmeter_tests.yaml

@@ -136,7 +136,7 @@ jobs:
           -Dpgsql_user="root" \
           -Dpgsql_pwd="public" \
           -Ddbname="mqtt" \
-          -Droute="apps/emqx_authn/test/data/certs" \
+          -Droute="apps/emqx_auth/test/data/certs" \
           -Dca_name="ca.crt" \
           -Dkey_name="client.key" \
           -Dcert_name="client.crt" \
@@ -199,7 +199,7 @@ jobs:
           -Dmysql_user="root" \
           -Dmysql_pwd="public" \
           -Ddbname="mqtt" \
-          -Droute="apps/emqx_authn/test/data/certs" \
+          -Droute="apps/emqx_auth/test/data/certs" \
           -Dca_name="ca.crt" \
           -Dkey_name="client.key" \
           -Dcert_name="client.crt" \

apps/emqx/src/emqx_hocon.erl

@@ -37,25 +37,25 @@ format_path([Name | Rest]) -> [iol(Name), "." | format_path(Rest)].
 %% @doc Plain check the input config.
 %% The input can either be `richmap' or plain `map'.
 %% Always return plain map with atom keys.
--spec check(module(), hocon:config() | iodata()) ->
+-spec check(hocon_schema:schema(), hocon:config() | iodata()) ->
     {ok, hocon:config()} | {error, any()}.
-check(SchemaModule, Conf) ->
+check(Schema, Conf) ->
     %% TODO: remove required
     %% fields should state required or not in their schema
     Opts = #{atom_key => true, required => false},
-    check(SchemaModule, Conf, Opts).
+    check(Schema, Conf, Opts).
 
-check(SchemaModule, Conf, Opts) when is_map(Conf) ->
+check(Schema, Conf, Opts) when is_map(Conf) ->
     try
-        {ok, hocon_tconf:check_plain(SchemaModule, Conf, Opts)}
+        {ok, hocon_tconf:check_plain(Schema, Conf, Opts)}
     catch
         throw:Errors:Stacktrace ->
             compact_errors(Errors, Stacktrace)
     end;
-check(SchemaModule, HoconText, Opts) ->
+check(Schema, HoconText, Opts) ->
     case hocon:binary(HoconText, #{format => map}) of
         {ok, MapConfig} ->
-            check(SchemaModule, MapConfig, Opts);
+            check(Schema, MapConfig, Opts);
         {error, Reason} ->
             {error, Reason}
     end.

apps/emqx/src/emqx_schema.erl

@@ -216,16 +216,18 @@ roots(high) ->
                 }
             )}
     ] ++
-        emqx_schema_hooks:injection_point('roots.high') ++
-        [
-            %% NOTE: authorization schema here is only to keep emqx app pure
-            %% the full schema for EMQX node is injected in emqx_conf_schema.
-            {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME,
-                sc(
-                    ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME),
-                    #{importance => ?IMPORTANCE_HIDDEN}
-                )}
-        ];
+        emqx_schema_hooks:injection_point(
+            'roots.high',
+            [
+                %% NOTE: authorization schema here is only to keep emqx app pure
+                %% the full schema for EMQX node is injected in emqx_conf_schema.
+                {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME,
+                    sc(
+                        ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME),
+                        #{importance => ?IMPORTANCE_HIDDEN}
+                    )}
+            ]
+        );
 roots(medium) ->
     [
         {"broker",

apps/emqx/src/emqx_schema_hooks.erl

@@ -22,14 +22,19 @@
     #{
         hookpoint() => [hocon_schema:field()]
     }.
--optional_callbacks([injected_fields/0]).
+-callback injected_fields(term()) ->
+    #{
+        hookpoint() => [hocon_schema:field()]
+    }.
+-optional_callbacks([injected_fields/0, injected_fields/1]).
 
 -export_type([hookpoint/0]).
 
 -define(HOOKPOINT_PT_KEY(POINT_NAME), {?MODULE, fields, POINT_NAME}).
 
 -export([
     injection_point/1,
+    injection_point/2,
     inject_from_modules/1
 ]).
 
@@ -43,9 +48,15 @@
 %% API
 %%--------------------------------------------------------------------
 
+-spec injection_point(hookpoint()) -> [hocon_schema:field()].
 injection_point(PointName) ->
-    persistent_term:get(?HOOKPOINT_PT_KEY(PointName), []).
+    injection_point(PointName, []).
+
+-spec injection_point(hookpoint(), [hocon_schema:field()]) -> [hocon_schema:field()].
+injection_point(PointName, Default) ->
+    persistent_term:get(?HOOKPOINT_PT_KEY(PointName), Default).
 
+-spec erase_injections() -> ok.
 erase_injections() ->
     lists:foreach(
         fun
@@ -57,6 +68,7 @@ erase_injections() ->
         persistent_term:get()
     ).
 
+-spec any_injections() -> boolean().
 any_injections() ->
     lists:any(
         fun
@@ -68,6 +80,7 @@ any_injections() ->
         persistent_term:get()
     ).
 
+-spec inject_from_modules([module() | {module(), term()}]) -> ok.
 inject_from_modules(Modules) ->
     Injections =
         lists:foldl(
@@ -83,6 +96,8 @@ inject_from_modules(Modules) ->
 
 append_module_injections(Module, AllInjections) when is_atom(Module) ->
     append_module_injections(Module:injected_fields(), AllInjections);
+append_module_injections({Module, Options}, AllInjections) when is_atom(Module) ->
+    append_module_injections(Module:injected_fields(Options), AllInjections);
 append_module_injections(ModuleInjections, AllInjections) when is_map(ModuleInjections) ->
     maps:fold(
         fun(PointName, Fields, Acc) ->

apps/emqx/test/emqx_common_test_helpers.erl

@@ -39,7 +39,8 @@
     flush/1,
     load/1,
     render_and_load_app_config/1,
-    render_and_load_app_config/2
+    render_and_load_app_config/2,
+    copy_acl_conf/0
 ]).
 
 -export([
@@ -527,11 +528,11 @@ copy_certs(_, _) ->
 
 copy_acl_conf() ->
     Dest = filename:join([code:lib_dir(emqx), "etc/acl.conf"]),
-    case code:lib_dir(emqx_authz) of
+    case code:lib_dir(emqx_auth) of
         {error, bad_name} ->
             (not filelib:is_regular(Dest)) andalso file:write_file(Dest, <<"">>);
         _ ->
-            {ok, _} = file:copy(deps_path(emqx_authz, "etc/acl.conf"), Dest)
+            {ok, _} = file:copy(deps_path(emqx_auth, "etc/acl.conf"), Dest)
     end,
     ok.
 

apps/emqx/test/emqx_cth_suite.erl

@@ -330,7 +330,7 @@ default_appspec(emqx, SuiteOpts) ->
         % overwrite everything with a default configuration.
         before_start => fun inhibit_config_loader/2
     };
-default_appspec(emqx_authz, _SuiteOpts) ->
+default_appspec(emqx_auth, _SuiteOpts) ->
     #{
         config => #{
             % NOTE
@@ -356,15 +356,21 @@ default_appspec(emqx_conf, SuiteOpts) ->
         Config,
         [
             emqx,
-            emqx_authz
+            emqx_auth
         ]
     ),
     #{
         config => SharedConfig,
-        % NOTE
-        % We inform `emqx` of our config loader before starting `emqx_conf` so that it won't
-        % overwrite everything with a default configuration.
-        before_start => fun inhibit_config_loader/2
+        before_start => fun(App, Conf) ->
+            % NOTE
+            % We inform `emqx` of our config loader before starting `emqx_conf` so that it won't
+            % overwrite everything with a default configuration.
+            ok = inhibit_config_loader(App, Conf),
+            % NOTE
+            % This should be done to pass authz schema validations.
+            % In production, acl.conf file is created by the release process.
+            ok = emqx_common_test_helpers:copy_acl_conf()
+        end
     };
 default_appspec(emqx_dashboard, _SuiteOpts) ->
     #{

apps/emqx_auth/include/emqx_auth.hrl

@@ -0,0 +1,22 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-ifndef(EMQX_AUTH_HRL).
+-define(EMQX_AUTH_HRL, true).
+
+-define(APP, emqx_auth).
+
+-endif.

apps/emqx_auth/include/emqx_auth_file.hrl

@@ -0,0 +1,23 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-ifndef(EMQX_AUTH_FILE_HRL).
+-define(EMQX_AUTH_FILE_HRL, true).
+
+-define(AUTHZ_TYPE, file).
+-define(AUTHZ_TYPE_BIN, <<"file">>).
+
+-endif.

apps/emqx_authn/include/emqx_authn.hrl → apps/emqx_auth/include/emqx_authn.hrl

@@ -17,23 +17,19 @@
 -ifndef(EMQX_AUTHN_HRL).
 -define(EMQX_AUTHN_HRL, true).
 
--include_lib("emqx_authentication.hrl").
+-include("emqx_authn_chains.hrl").
 
--define(APP, emqx_authn).
-
--define(AUTHN, emqx_authentication).
+-define(AUTHN, emqx_authn_chains).
 
 -define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}").
 
--define(AUTH_SHARD, emqx_authn_shard).
-
 %% has to be the same as the root field name defined in emqx_schema
 -define(CONF_NS, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME).
 -define(CONF_NS_ATOM, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM).
 -define(CONF_NS_BINARY, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY).
 
 -type authenticator_id() :: binary().
 
--define(RESOURCE_GROUP, <<"emqx_authn">>).
+-define(AUTHN_RESOURCE_GROUP, <<"emqx_authn">>).
 
 -endif.

apps/emqx_authn/include/emqx_authentication.hrl → apps/emqx_auth/include/emqx_authn_chains.hrl

@@ -14,8 +14,8 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--ifndef(EMQX_AUTHENTICATION_HRL).
--define(EMQX_AUTHENTICATION_HRL, true).
+-ifndef(EMQX_AUTHN_CHAINS_HRL).
+-define(EMQX_AUTHN_CHAINS_HRL, true).
 
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx/include/emqx_access_control.hrl").

apps/emqx_authz/include/emqx_authz.hrl → apps/emqx_auth/include/emqx_authz.hrl

@@ -16,7 +16,7 @@
 
 -include_lib("emqx/include/emqx_access_control.hrl").
 
--define(APP, emqx_authz).
+-include("emqx_auth.hrl").
 
 %% authz_mnesia
 -define(ACL_TABLE, emqx_acl).
@@ -157,7 +157,7 @@
     count => 1
 }).
 
--define(RESOURCE_GROUP, <<"emqx_authz">>).
+-define(AUTHZ_RESOURCE_GROUP, <<"emqx_authz">>).
 
 -define(AUTHZ_FEATURES, [rich_actions]).
 

apps/emqx_authn/rebar.config → apps/emqx_auth/rebar.config

@@ -2,12 +2,7 @@
 
 {deps, [
     {emqx, {path, "../emqx"}},
-    {emqx_utils, {path, "../emqx_utils"}},
-    {emqx_connector, {path, "../emqx_connector"}},
-    {emqx_mongodb, {path, "../emqx_mongodb"}},
-    {emqx_redis, {path, "../emqx_redis"}},
-    {emqx_mysql, {path, "../emqx_mysql"}},
-    {emqx_bridge_http, {path, "../emqx_bridge_http"}}
+    {emqx_utils, {path, "../emqx_utils"}}
 ]}.
 
 {edoc_opts, [{preprocess, true}]}.
@@ -34,6 +29,4 @@
 {cover_opts, [verbose]}.
 {cover_export_enabled, true}.
 
-{erl_first_files, ["src/emqx_authentication.erl"]}.
-
 {project_plugins, [erlfmt]}.

apps/emqx_auth/src/emqx_auth.app.src

@@ -0,0 +1,17 @@
+%% -*- mode: erlang -*-
+{application, emqx_auth, [
+    {description, "EMQX Authentication and authorization"},
+    {vsn, "0.1.27"},
+    {modules, []},
+    {registered, [emqx_auth_sup]},
+    {applications, [
+        kernel,
+        stdlib,
+        emqx
+    ]},
+    {mod, {emqx_auth_app, []}},
+    {env, []},
+    {licenses, ["Apache-2.0"]},
+    {maintainers, ["EMQX Team <contact@emqx.io>"]},
+    {links, [{"Homepage", "https://emqx.io/"}]}
+]}.