diff --git a/big_tests/tests/gdpr_SUITE.erl b/big_tests/tests/gdpr_SUITE.erl index 2eea2f8fa21..dd9f5362bd0 100644 --- a/big_tests/tests/gdpr_SUITE.erl +++ b/big_tests/tests/gdpr_SUITE.erl @@ -44,32 +44,39 @@ suite() -> all() -> [ {group, retrieve_personal_data}, - {group, retrieve_personal_data_pubsub}, - {group, data_is_not_retrieved_for_missing_user} + {group, retrieve_personal_data_with_mods_disabled}, + {group, retrieve_negative} ]. groups() -> + %% **DON'T** make any of these groups parallel, because calling mongooseimctl + %% in parallel is broken! [ - {retrieve_personal_data, [parallel], [ - % per type - retrieve_vcard, - %retrieve_roster, - %retrieve_mam, - %retrieve_offline, - %retrieve_private_xml, - %retrieve_inbox, - retrieve_logs - ]}, - {retrieve_personal_data_pubsub, [], [ - retrieve_pubsub_payloads, - dont_retrieve_other_user_pubsub_payload, - retrieve_pubsub_subscriptions, - retrieve_created_pubsub_nodes, - retrieve_all_pubsub_data - ]}, - {data_is_not_retrieved_for_missing_user, [], - [data_is_not_retrieved_for_missing_user] - } + {retrieve_personal_data, [], [ + retrieve_vcard, + %retrieve_roster, + %retrieve_mam, + %retrieve_offline, + %retrieve_private_xml, + %retrieve_inbox, + retrieve_logs, + {group, retrieve_personal_data_pubsub} + ]}, + {retrieve_personal_data_pubsub, [], [ + retrieve_pubsub_payloads, + dont_retrieve_other_user_pubsub_payload, + retrieve_pubsub_subscriptions, + retrieve_created_pubsub_nodes, + retrieve_all_pubsub_data + ]}, + {retrieve_personal_data_with_mods_disabled, [], [ + retrieve_vcard, + retrieve_logs, + retrieve_all_pubsub_data + ]}, + {retrieve_negative, [], [ + data_is_not_retrieved_for_missing_user + ]} ]. init_per_suite(Config) -> @@ -82,6 +89,12 @@ end_per_suite(Config) -> escalus_fresh:clean(), escalus:end_per_suite(Config). +init_per_group(retrieve_personal_data_with_mods_disabled, Config) -> + dynamic_modules:ensure_modules(domain(), pubsub_required_modules()), + [{disable_module, true} | Config]; +init_per_group(retrieve_personal_data_pubsub, Config) -> + dynamic_modules:ensure_modules(domain(), pubsub_required_modules()), + Config; init_per_group(_GN, Config) -> Config. @@ -113,7 +126,6 @@ init_per_testcase(retrieve_mam = CN, Config) -> escalus:init_per_testcase(CN, Config) end; init_per_testcase(CN, Config) -> - dynamic_modules:ensure_modules(domain(), pubsub_required_modules()), escalus:init_per_testcase(CN, Config). end_per_testcase(CN, Config) -> @@ -168,6 +180,7 @@ retrieve_vcard(Config) -> "vcard" => [{contains, "Alice"}, {contains, "Ecila"}] } ], + maybe_stop_and_unload_module(mod_vcard, mod_vcard_backend, Config), retrieve_and_validate_personal_data( Alice, Config, "vcard", ExpectedHeader, ExpectedItems) end). @@ -179,6 +192,7 @@ retrieve_roster(Config) -> ExpectedItems = [ #{ "jid" => escalus_client:short_jid(Bob) } ], + maybe_stop_and_unload_module(mod_roster, mod_roster_backend, Config), retrieve_and_validate_personal_data( Alice, Config, "roster", ExpectedHeader, ExpectedItems) end). @@ -205,6 +219,7 @@ retrieve_offline(Config) -> ExpectedItems = [ #{ "packet" => [{contains, Body}], "from" => BobJid } ], + maybe_stop_and_unload_module(mod_offline, mod_offline_backend, Config), retrieve_and_validate_personal_data( Alice, Config, "offline", ExpectedHeader, ExpectedItems) end). @@ -227,6 +242,8 @@ retrieve_pubsub_payloads(Config) -> pubsub_payloads_row_map(NodeName1, "Item2",StringItem2), pubsub_payloads_row_map(NodeName1, "Item3", StringItem3), pubsub_payloads_row_map(NodeName2, "OtherItem", StringOther)], + + maybe_stop_and_unload_module(mod_vcard, mod_vcard_backend, Config), retrieve_and_validate_personal_data( Alice, Config, "pubsub_payloads", ["node_name", "item_id", "payload"], ExpectedItems) end). @@ -320,6 +337,7 @@ retrieve_all_pubsub_data(Config) -> pubsub_tools:receive_item_notification(Bob, <<"Item2">>, Node2, []), pubsub_tools:publish(Bob, <<"Item3">>, Node1, [{with_payload, {true, BinItem3}}]), + maybe_stop_and_unload_module(mod_pubsub, mod_pubsub_db_backend, Config), %% Bob has one subscription, one node created and one payload sent retrieve_and_validate_personal_data( Bob, Config, "pubsub_subscriptions", ["node_name"], @@ -341,11 +359,8 @@ retrieve_all_pubsub_data(Config) -> retrieve_and_validate_personal_data( Alice, Config, "pubsub_payloads", ["node_name", "item_id","payload"], [pubsub_payloads_row_map(NodeName1, "Item1", StringItem1), - pubsub_payloads_row_map(NodeName2, "Item2", StringItem2)]), - - Nodes = [{Alice, Node1}, {Alice, Node2}, {Bob, Node3}], - [pubsub_tools:delete_node(User, Node, []) || {User, Node} <- Nodes] - end). + pubsub_payloads_row_map(NodeName2, "Item2", StringItem2)]) + end). retrieve_private_xml(Config) -> @@ -412,6 +427,16 @@ data_is_not_retrieved_for_missing_user(Config) -> domain() -> <<"localhost">>. % TODO: Make dynamic? +maybe_stop_and_unload_module(Module, BackendProxy, Config) -> + case proplists:get_value(disable_module, Config) of + true -> + dynamic_modules:stop(domain(), Module), + mongoose_helper:successful_rpc(code, purge, [BackendProxy]), + true = mongoose_helper:successful_rpc(code, delete, [BackendProxy]); + _ -> + ok + end. + retrieve_and_validate_personal_data(Alice, Config, FilePrefix, ExpectedHeader, ExpectedItems) -> PersonalCSV = retrieve_and_decode_personal_data(Alice, Config, FilePrefix), PersonalMaps = csv_to_maps(ExpectedHeader, PersonalCSV), diff --git a/src/admin_extra/service_admin_extra_gdpr.erl b/src/admin_extra/service_admin_extra_gdpr.erl index 314f1a2383a..681b3b34657 100644 --- a/src/admin_extra/service_admin_extra_gdpr.erl +++ b/src/admin_extra/service_admin_extra_gdpr.erl @@ -56,10 +56,7 @@ retrieve_all(Username, Domain, ResultFilePath) -> -spec modules_with_personal_data() -> [module()]. modules_with_personal_data() -> - [ - mod_vcard, - mod_pubsub - ]. + mongoose_lib:find_behaviour_implementations(gdpr). -spec get_data_from_modules(jid:user(), jid:server()) -> [{gdpr:data_group(), gdpr:schema(), gdpr:entries()}]. diff --git a/src/mod_vcard.erl b/src/mod_vcard.erl index 72926ffd446..fe29b7bf75c 100644 --- a/src/mod_vcard.erl +++ b/src/mod_vcard.erl @@ -134,12 +134,17 @@ get_personal_data(Username, Server) -> LServer = jid:nameprep(Server), Jid = jid:to_binary({LUser, LServer}), Schema = ["jid", "vcard"], - Entries = case mod_vcard_backend:get_vcard(LUser, LServer) of - {ok, Record} -> - SerializedRecords = exml:to_binary(Record), - [{Jid, SerializedRecords}]; - _ -> [] - end, + Entries = lists:flatmap(fun(B) -> + try B:get_vcard(LUser, LServer) of + {ok, Record} -> + SerializedRecords = exml:to_binary(Record), + [{Jid, SerializedRecords}]; + _ -> [] + catch + _:_ -> + [] + end + end, mongoose_lib:find_behaviour_implementations(mod_vcard)), [{vcard, Schema, Entries}]. -spec default_search_fields() -> list(). diff --git a/src/mongoose_lib.erl b/src/mongoose_lib.erl index e2f17164a9f..899fc4e417d 100644 --- a/src/mongoose_lib.erl +++ b/src/mongoose_lib.erl @@ -1,6 +1,7 @@ -module(mongoose_lib). --export([bin_to_int/1, log_if_backend_error/4]). +-export([find_behaviour_implementations/1]). +-export([log_if_backend_error/4]). %% Maps -export([maps_append/3]). -export([maps_foreach/2]). @@ -9,15 +10,29 @@ -include("mongoose.hrl"). -%% @doc string:to_integer/1 for binaries -bin_to_int(Bin) -> - bin_to_int(Bin, 0). +%% ------------------------------------------------------------------ +%% Behaviour util +%% ------------------------------------------------------------------ -bin_to_int(<>, X) when $0 =< H, H =< $9 -> - bin_to_int(T, (X*10)+(H-$0)); -bin_to_int(Bin, X) -> - {X, Bin}. +%% WARNING! For simplicity, this function searches only MongooseIM code dir +-spec find_behaviour_implementations(Behaviour :: module()) -> [module()]. +find_behaviour_implementations(Behaviour) -> + {ok, EbinFiles} = file:list_dir(code:lib_dir(mongooseim, ebin)), + Mods = [ list_to_atom(filename:rootname(File)) + || File <- EbinFiles, filename:extension(File) == ".beam" ], + lists:filter(fun(M) -> + try lists:keyfind([Behaviour], 2, M:module_info(attributes)) of + {behavior, _} -> true; + {behaviour, _} -> true; + _ -> false + catch + _:_ -> false + end + end, Mods). +%% ------------------------------------------------------------------ +%% Logging +%% ------------------------------------------------------------------ %% @doc Database backends for various modules return ok, {atomic, ok} %% or {atomic, []} on success, and usually {error, ...} on failure. diff --git a/src/pubsub/mod_pubsub.erl b/src/pubsub/mod_pubsub.erl index 9fade2d9355..164a1706632 100644 --- a/src/pubsub/mod_pubsub.erl +++ b/src/pubsub/mod_pubsub.erl @@ -262,14 +262,24 @@ process_packet(Acc, From, To, El, #state{server_host = ServerHost, access = Acce get_personal_data(Username, Server) -> LUser = jid:nodeprep(Username), LServer = jid:nodeprep(Server), - Payloads = mod_pubsub_db_backend:get_user_payloads(LUser, LServer), - Nodes = mod_pubsub_db_backend:get_user_nodes(LUser, LServer), - Subscriptions = mod_pubsub_db_backend:get_user_subscriptions(LUser, LServer), + Backends = mongoose_lib:find_behaviour_implementations(mod_pubsub_db), + Payloads = get_personal_data_group(LUser, LServer, Backends, get_user_payloads), + Nodes = get_personal_data_group(LUser, LServer, Backends, get_user_nodes), + Subscriptions = get_personal_data_group(LUser, LServer, Backends, get_user_subscriptions), [{pubsub_payloads, ["node_name", "item_id", "payload"], Payloads}, {pubsub_nodes, ["node_name", "type"], Nodes}, {pubsub_subscriptions, ["node_name"], Subscriptions}]. +get_personal_data_group(LUser, LServer, Backends, FunctionName) -> + lists:flatmap(fun(B) -> try B:FunctionName(LUser, LServer) of + Result when is_list(Result) -> Result; + _ -> [] + catch + _:_ -> [] + end + end, Backends). + %%==================================================================== %% gen_server callbacks %%====================================================================