feat: implement configuration and user data export/import

apps/emqx/src/bhvrs/emqx_config_backup.erl

@@ -0,0 +1,24 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_config_backup).
+
+-callback import_config(RawConf :: map()) ->
+    {ok, #{
+        root_key => emqx_utils_maps:config_key(),
+        changed => [emqx_utils_maps:config_path()]
+    }}
+    | {error, #{root_key => emqx_utils_maps:config_key(), reason => term()}}.

apps/emqx/src/bhvrs/emqx_db_backup.erl

@@ -0,0 +1,19 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_db_backup).
+
+-callback backup_tables() -> [mria:table()].

apps/emqx/src/emqx_banned.erl

@@ -17,6 +17,7 @@
 -module(emqx_banned).
 
 -behaviour(gen_server).
+-behaviour(emqx_db_backup).
 
 -include("emqx.hrl").
 -include("logger.hrl").
@@ -50,6 +51,8 @@
     code_change/3
 ]).
 
+-export([backup_tables/0]).
+
 %% Internal exports (RPC)
 -export([
     expire_banned_items/1
@@ -82,6 +85,11 @@ mnesia(boot) ->
         {storage_properties, [{ets, [{read_concurrency, true}]}]}
     ]).
 
+%%--------------------------------------------------------------------
+%% Data backup
+%%--------------------------------------------------------------------
+backup_tables() -> [?BANNED_TAB].
+
 %% @doc Start the banned server.
 -spec start_link() -> startlink_ret().
 start_link() ->

apps/emqx/src/emqx_listeners.erl

@@ -843,7 +843,9 @@ convert_certs(ListenerConf) ->
             Listeners1 =
                 maps:fold(
                     fun(Name, Conf, Acc1) ->
-                        Acc1#{Name => convert_certs(Type, Name, Conf)}
+                        Conf1 = convert_certs(Type, Name, Conf),
+                        Conf2 = convert_authn_certs(Type, Name, Conf1),
+                        Acc1#{Name => Conf2}
                     end,
                     #{},
                     Listeners0
@@ -866,6 +868,19 @@ convert_certs(Type, Name, Conf) ->
             throw({bad_ssl_config, Reason})
     end.
 
+convert_authn_certs(Type, Name, #{<<"authentication">> := AuthNList} = Conf) ->
+    ChainName = listener_id(Type, Name),
+    AuthNList1 = lists:map(
+        fun(AuthN) ->
+            CertsDir = emqx_authentication_config:certs_dir(ChainName, AuthN),
+            emqx_authentication_config:convert_certs(CertsDir, AuthN)
+        end,
+        AuthNList
+    ),
+    Conf#{<<"authentication">> => AuthNList1};
+convert_authn_certs(_Type, _Name, Conf) ->
+    Conf.
+
 filter_stacktrace({Reason, _Stacktrace}) -> Reason;
 filter_stacktrace(Reason) -> Reason.
 

apps/emqx/src/emqx_ssl_crl_cache.erl

@@ -33,11 +33,11 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%----------------------------------------------------------------------
-% Based on `otp/lib/ssl/src/ssl_crl_cache.erl'
-%----------------------------------------------------------------------
+%%----------------------------------------------------------------------
+%% Based on `otp/lib/ssl/src/ssl_crl_cache.erl'
+%%----------------------------------------------------------------------
 
-%----------------------------------------------------------------------
+%%----------------------------------------------------------------------
 %% Purpose: Simple default CRL cache
 %%----------------------------------------------------------------------
 

apps/emqx/test/emqx_common_test_helpers.erl

@@ -859,6 +859,12 @@ setup_node(Node, Opts) when is_map(Opts) ->
     %% Setting env before starting any applications
     set_envs(Node, Env),
 
+    NodeDataDir = filename:join([
+        PrivDataDir,
+        node(),
+        integer_to_list(erlang:unique_integer())
+    ]),
+
     %% Here we start the apps
     EnvHandlerForRpc =
         fun(App) ->
@@ -870,17 +876,10 @@ setup_node(Node, Opts) when is_map(Opts) ->
                     %% to avoid sharing data between executions and/or
                     %% nodes.  these variables might not be in the
                     %% config file (e.g.: emqx_enterprise_schema).
-                    NodeDataDir = filename:join([
-                        PrivDataDir,
-                        node(),
-                        integer_to_list(erlang:unique_integer())
-                    ]),
                     Cookie = atom_to_list(erlang:get_cookie()),
                     os:putenv("EMQX_NODE__DATA_DIR", NodeDataDir),
                     os:putenv("EMQX_NODE__COOKIE", Cookie),
                     emqx_config:init_load(SchemaMod),
-                    os:unsetenv("EMQX_NODE__DATA_DIR"),
-                    os:unsetenv("EMQX_NODE__COOKIE"),
                     application:set_env(emqx, init_config_load_done, true)
                 end,
 

apps/emqx_authn/src/emqx_authn.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_authn).
 
+-behaviour(emqx_config_backup).
+
 -export([
     providers/0,
     check_config/1,
@@ -24,6 +26,11 @@
     get_enabled_authns/0
 ]).
 
+%% Data backup
+-export([
+    import_config/1
+]).
+
 -include("emqx_authn.hrl").
 
 providers() ->
@@ -126,3 +133,32 @@ get_enabled_authns() ->
 
 tally_authenticators(#{id := AuthenticatorName}, Acc) ->
     maps:update_with(AuthenticatorName, fun(N) -> N + 1 end, 1, Acc).
+
+%%------------------------------------------------------------------------------
+%% Data backup
+%%------------------------------------------------------------------------------
+
+-define(IMPORT_OPTS, #{override_to => cluster}).
+
+import_config(RawConf) ->
+    AuthnList = authn_list(maps:get(?CONF_NS_BINARY, RawConf, [])),
+    OldAuthnList = emqx:get_raw_config([?CONF_NS_BINARY], []),
+    MergedAuthnList = emqx_utils:merge_lists(
+        OldAuthnList, AuthnList, fun emqx_authentication:authenticator_id/1
+    ),
+    case emqx_conf:update([?CONF_NS_ATOM], MergedAuthnList, ?IMPORT_OPTS) of
+        {ok, #{raw_config := NewRawConf}} ->
+            {ok, #{root_key => ?CONF_NS_ATOM, changed => changed_paths(OldAuthnList, NewRawConf)}};
+        Error ->
+            {error, #{root_key => ?CONF_NS_ATOM, reason => Error}}
+    end.
+
+changed_paths(OldAuthnList, NewAuthnList) ->
+    KeyFun = fun emqx_authentication:authenticator_id/1,
+    Changed = maps:get(changed, emqx_utils:diff_lists(NewAuthnList, OldAuthnList, KeyFun)),
+    [[?CONF_NS_BINARY, emqx_authentication:authenticator_id(OldAuthn)] || {OldAuthn, _} <- Changed].
+
+authn_list(Authn) when is_list(Authn) ->
+    Authn;
+authn_list(Authn) when is_map(Authn) ->
+    [Authn].

apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl

@@ -22,6 +22,7 @@
 
 -behaviour(hocon_schema).
 -behaviour(emqx_authentication).
+-behaviour(emqx_db_backup).
 
 -export([
     namespace/0,
@@ -54,6 +55,8 @@
     group_match_spec/1
 ]).
 
+-export([backup_tables/0]).
+
 %% Internal exports (RPC)
 -export([
     do_destroy/1,
@@ -101,6 +104,12 @@ mnesia(boot) ->
         {storage_properties, [{ets, [{read_concurrency, true}]}]}
     ]).
 
+%%------------------------------------------------------------------------------
+%% Data backup
+%%------------------------------------------------------------------------------
+
+backup_tables() -> [?TAB].
+
 %%------------------------------------------------------------------------------
 %% Hocon Schema
 %%------------------------------------------------------------------------------
@@ -357,6 +366,9 @@ check_client_final_message(Bin, #{is_superuser := IsSuperuser} = Cache, #{algori
 
 add_user(UserGroup, UserID, Password, IsSuperuser, State) ->
     {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(Password, State),
+    write_user(UserGroup, UserID, StoredKey, ServerKey, Salt, IsSuperuser).
+
+write_user(UserGroup, UserID, StoredKey, ServerKey, Salt, IsSuperuser) ->
     UserInfo = #user_info{
         user_id = {UserGroup, UserID},
         stored_key = StoredKey,

apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl

@@ -23,6 +23,7 @@
 
 -behaviour(hocon_schema).
 -behaviour(emqx_authentication).
+-behaviour(emqx_db_backup).
 
 -export([
     namespace/0,
@@ -66,6 +67,10 @@
     import_csv/3
 ]).
 
+-export([mnesia/1]).
+
+-export([backup_tables/0]).
+
 -type user_group() :: binary().
 -type user_id() :: binary().
 
@@ -76,8 +81,6 @@
     is_superuser :: boolean()
 }).
 
--export([mnesia/1]).
-
 -boot_mnesia({mnesia, [boot]}).
 
 -define(TAB, ?MODULE).
@@ -103,6 +106,11 @@ mnesia(boot) ->
         {storage_properties, [{ets, [{read_concurrency, true}]}]}
     ]).
 
+%%------------------------------------------------------------------------------
+%% Data backup
+%%------------------------------------------------------------------------------
+backup_tables() -> [?TAB].
+
 %%------------------------------------------------------------------------------
 %% Hocon Schema
 %%------------------------------------------------------------------------------

apps/emqx_authz/src/emqx_authz.erl

@@ -15,7 +15,9 @@
 %%--------------------------------------------------------------------
 
 -module(emqx_authz).
+
 -behaviour(emqx_config_handler).
+-behaviour(emqx_config_backup).
 
 -include("emqx_authz.hrl").
 -include_lib("emqx/include/logger.hrl").
@@ -44,6 +46,13 @@
 
 -export([acl_conf_file/0]).
 
+%% Data backup
+-export([
+    import_config/1,
+    maybe_read_acl_file/1,
+    maybe_write_acl_file/1
+]).
+
 -type source() :: map().
 
 -type match_result() :: {matched, allow} | {matched, deny} | nomatch.
@@ -326,9 +335,9 @@ init_metrics(Source) ->
             )
     end.
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% AuthZ callbacks
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 %% @doc Check AuthZ
 -spec authorize(
@@ -451,9 +460,58 @@ do_authorize(
 get_enabled_authzs() ->
     lists:usort([Type || #{type := Type, enable := true} <- lookup()]).
 
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
+%% Data backup
+%%------------------------------------------------------------------------------
+
+import_config(#{?CONF_NS_BINARY := AuthzConf}) ->
+    Sources = maps:get(<<"sources">>, AuthzConf, []),
+    OldSources = emqx:get_raw_config(?CONF_KEY_PATH, []),
+    MergedSources = emqx_utils:merge_lists(OldSources, Sources, fun type/1),
+    MergedAuthzConf = AuthzConf#{<<"sources">> => MergedSources},
+    case emqx_conf:update([?CONF_NS_ATOM], MergedAuthzConf, #{override_to => cluster}) of
+        {ok, #{raw_config := #{<<"sources">> := NewSources}}} ->
+            {ok, #{
+                root_key => ?CONF_NS_ATOM,
+                changed => changed_paths(OldSources, NewSources)
+            }};
+        Error ->
+            {error, #{root_key => ?CONF_NS_ATOM, reason => Error}}
+    end;
+import_config(_RawConf) ->
+    {ok, #{root_key => ?CONF_NS_ATOM, changed => []}}.
+
+changed_paths(OldSources, NewSources) ->
+    Changed = maps:get(changed, emqx_utils:diff_lists(NewSources, OldSources, fun type/1)),
+    [?CONF_KEY_PATH ++ [type(OldSource)] || {OldSource, _} <- Changed].
+
+maybe_read_acl_file(RawConf) ->
+    maybe_convert_acl_file(RawConf, fun read_acl_file/1).
+
+maybe_write_acl_file(RawConf) ->
+    maybe_convert_acl_file(RawConf, fun write_acl_file/1).
+
+maybe_convert_acl_file(
+    #{?CONF_NS_BINARY := #{<<"sources">> := Sources} = AuthRawConf} = RawConf, Fun
+) ->
+    Sources1 = lists:map(
+        fun
+            (#{<<"type">> := <<"file">>} = FileSource) -> Fun(FileSource);
+            (Source) -> Source
+        end,
+        Sources
+    ),
+    RawConf#{?CONF_NS_BINARY => AuthRawConf#{<<"sources">> => Sources1}};
+maybe_convert_acl_file(RawConf, _Fun) ->
+    RawConf.
+
+read_acl_file(#{<<"path">> := Path} = Source) ->
+    {ok, Rules} = emqx_authz_file:read_file(Path),
+    maps:remove(<<"path">>, Source#{<<"rules">> => Rules}).
+
+%%------------------------------------------------------------------------------
 %% Internal function
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
 client_info_source() ->
     emqx_authz_client_info:create(