Dynamic lookup of GDPR-related modules for retrieval (#2297)

* Find GDPR modules dynamically

* Find module backends for GDPR retrieval dynamically

* Fix behaviours' impls lookup

* Add test case for GDPR retrieval with disabled modules

* Fix GDPR suite

big_tests/tests/gdpr_SUITE.erl

@@ -44,32 +44,39 @@ suite() ->
 all() ->
     [
      {group, retrieve_personal_data},
-     {group, retrieve_personal_data_pubsub},
-     {group, data_is_not_retrieved_for_missing_user}
+     {group, retrieve_personal_data_with_mods_disabled},
+     {group, retrieve_negative}
     ].
 
 groups() ->
+    %% **DON'T** make any of these groups parallel, because calling mongooseimctl
+    %% in parallel is broken!
     [
-     {retrieve_personal_data, [parallel], [
-                                   % per type
-                                   retrieve_vcard,
-                                   %retrieve_roster,
-                                   %retrieve_mam,
-                                   %retrieve_offline,
-                                   %retrieve_private_xml,
-                                   %retrieve_inbox,
-                                   retrieve_logs
-                                  ]},
-        {retrieve_personal_data_pubsub, [], [
-            retrieve_pubsub_payloads,
-            dont_retrieve_other_user_pubsub_payload,
-            retrieve_pubsub_subscriptions,
-            retrieve_created_pubsub_nodes,
-            retrieve_all_pubsub_data
-        ]},
-    {data_is_not_retrieved_for_missing_user, [],
-        [data_is_not_retrieved_for_missing_user]
-    }
+     {retrieve_personal_data, [], [
+                                           retrieve_vcard,
+                                           %retrieve_roster,
+                                           %retrieve_mam,
+                                           %retrieve_offline,
+                                           %retrieve_private_xml,
+                                           %retrieve_inbox,
+                                           retrieve_logs,
+                                           {group, retrieve_personal_data_pubsub}
+                                          ]},
+     {retrieve_personal_data_pubsub, [], [
+                                          retrieve_pubsub_payloads,
+                                          dont_retrieve_other_user_pubsub_payload,
+                                          retrieve_pubsub_subscriptions,
+                                          retrieve_created_pubsub_nodes,
+                                          retrieve_all_pubsub_data
+                                         ]},
+     {retrieve_personal_data_with_mods_disabled, [], [
+                                                      retrieve_vcard,
+                                                      retrieve_logs,
+                                                      retrieve_all_pubsub_data
+                                                     ]},
+     {retrieve_negative, [], [
+                              data_is_not_retrieved_for_missing_user
+                             ]}
     ].
 
 init_per_suite(Config) ->
@@ -82,6 +89,12 @@ end_per_suite(Config) ->
     escalus_fresh:clean(),
     escalus:end_per_suite(Config).
 
+init_per_group(retrieve_personal_data_with_mods_disabled, Config) ->
+    dynamic_modules:ensure_modules(domain(), pubsub_required_modules()),
+    [{disable_module, true} | Config];
+init_per_group(retrieve_personal_data_pubsub, Config) ->
+    dynamic_modules:ensure_modules(domain(), pubsub_required_modules()),
+    Config;
 init_per_group(_GN, Config) ->
     Config.
 
@@ -113,7 +126,6 @@ init_per_testcase(retrieve_mam = CN, Config) ->
             escalus:init_per_testcase(CN, Config)
     end;
 init_per_testcase(CN, Config) ->
-    dynamic_modules:ensure_modules(domain(), pubsub_required_modules()),
     escalus:init_per_testcase(CN, Config).
 
 end_per_testcase(CN, Config) ->
@@ -168,6 +180,7 @@ retrieve_vcard(Config) ->
                                 "vcard" => [{contains, "Alice"},
                                             {contains, "Ecila"}] }
                             ],
+            maybe_stop_and_unload_module(mod_vcard, mod_vcard_backend, Config),
             retrieve_and_validate_personal_data(
               Alice, Config, "vcard", ExpectedHeader, ExpectedItems)
         end).
@@ -179,6 +192,7 @@ retrieve_roster(Config) ->
             ExpectedItems = [
                              #{ "jid" => escalus_client:short_jid(Bob) }
                             ],
+            maybe_stop_and_unload_module(mod_roster, mod_roster_backend, Config),
             retrieve_and_validate_personal_data(
               Alice, Config, "roster", ExpectedHeader, ExpectedItems)
         end).
@@ -205,6 +219,7 @@ retrieve_offline(Config) ->
             ExpectedItems = [
                              #{ "packet" => [{contains, Body}], "from" => BobJid }
                             ],
+            maybe_stop_and_unload_module(mod_offline, mod_offline_backend, Config),
             retrieve_and_validate_personal_data(
               Alice, Config, "offline", ExpectedHeader, ExpectedItems)
         end).
@@ -227,6 +242,8 @@ retrieve_pubsub_payloads(Config) ->
                          pubsub_payloads_row_map(NodeName1, "Item2",StringItem2),
                          pubsub_payloads_row_map(NodeName1, "Item3", StringItem3),
                          pubsub_payloads_row_map(NodeName2, "OtherItem", StringOther)],
+
+        maybe_stop_and_unload_module(mod_vcard, mod_vcard_backend, Config),
         retrieve_and_validate_personal_data(
             Alice, Config, "pubsub_payloads", ["node_name", "item_id", "payload"], ExpectedItems)
                                               end).
@@ -320,6 +337,7 @@ retrieve_all_pubsub_data(Config) ->
         pubsub_tools:receive_item_notification(Bob, <<"Item2">>, Node2, []),
         pubsub_tools:publish(Bob, <<"Item3">>, Node1, [{with_payload, {true, BinItem3}}]),
 
+        maybe_stop_and_unload_module(mod_pubsub, mod_pubsub_db_backend, Config),
         %% Bob has one subscription, one node created and one payload sent
         retrieve_and_validate_personal_data(
             Bob, Config, "pubsub_subscriptions", ["node_name"],
@@ -341,11 +359,8 @@ retrieve_all_pubsub_data(Config) ->
         retrieve_and_validate_personal_data(
             Alice, Config, "pubsub_payloads", ["node_name", "item_id","payload"],
             [pubsub_payloads_row_map(NodeName1, "Item1", StringItem1),
-             pubsub_payloads_row_map(NodeName2, "Item2", StringItem2)]),
-
-        Nodes = [{Alice, Node1}, {Alice, Node2}, {Bob, Node3}],
-        [pubsub_tools:delete_node(User, Node, []) || {User, Node} <- Nodes]
-                                                        end).
+             pubsub_payloads_row_map(NodeName2, "Item2", StringItem2)])
+      end).
 
 
 retrieve_private_xml(Config) ->
@@ -412,6 +427,16 @@ data_is_not_retrieved_for_missing_user(Config) ->
 domain() ->
     <<"localhost">>. % TODO: Make dynamic?
 
+maybe_stop_and_unload_module(Module, BackendProxy, Config) ->
+    case proplists:get_value(disable_module, Config) of
+        true ->
+            dynamic_modules:stop(domain(), Module),
+            mongoose_helper:successful_rpc(code, purge, [BackendProxy]),
+            true = mongoose_helper:successful_rpc(code, delete, [BackendProxy]);
+        _ ->
+            ok
+    end.
+
 retrieve_and_validate_personal_data(Alice, Config, FilePrefix, ExpectedHeader, ExpectedItems) ->
     PersonalCSV = retrieve_and_decode_personal_data(Alice, Config, FilePrefix),
     PersonalMaps = csv_to_maps(ExpectedHeader, PersonalCSV),

src/admin_extra/service_admin_extra_gdpr.erl

@@ -56,10 +56,7 @@ retrieve_all(Username, Domain, ResultFilePath) ->
 
 -spec modules_with_personal_data() -> [module()].
 modules_with_personal_data() ->
-    [
-     mod_vcard,
-     mod_pubsub
-    ].
+    mongoose_lib:find_behaviour_implementations(gdpr).
 
 -spec get_data_from_modules(jid:user(), jid:server()) ->
     [{gdpr:data_group(), gdpr:schema(), gdpr:entries()}].

src/mod_vcard.erl

@@ -134,12 +134,17 @@ get_personal_data(Username, Server) ->
     LServer = jid:nameprep(Server),
     Jid = jid:to_binary({LUser, LServer}),
     Schema = ["jid", "vcard"],
-    Entries = case mod_vcard_backend:get_vcard(LUser, LServer) of
-        {ok, Record} ->
-            SerializedRecords = exml:to_binary(Record),
-            [{Jid, SerializedRecords}];
-         _ -> []
-        end,
+    Entries = lists:flatmap(fun(B) ->
+                                    try B:get_vcard(LUser, LServer) of
+                                        {ok, Record} ->
+                                            SerializedRecords = exml:to_binary(Record),
+                                            [{Jid, SerializedRecords}];
+                                        _ -> []
+                                    catch
+                                        _:_ ->
+                                            []
+                                    end
+                            end, mongoose_lib:find_behaviour_implementations(mod_vcard)),
     [{vcard, Schema, Entries}].
 
 -spec default_search_fields() -> list().

src/mongoose_lib.erl

@@ -1,6 +1,7 @@
 -module(mongoose_lib).
--export([bin_to_int/1, log_if_backend_error/4]).
 
+-export([find_behaviour_implementations/1]).
+-export([log_if_backend_error/4]).
 %% Maps
 -export([maps_append/3]).
 -export([maps_foreach/2]).
@@ -9,15 +10,29 @@
 
 -include("mongoose.hrl").
 
-%% @doc string:to_integer/1 for binaries
-bin_to_int(Bin) ->
-    bin_to_int(Bin, 0).
+%% ------------------------------------------------------------------
+%% Behaviour util
+%% ------------------------------------------------------------------
 
-bin_to_int(<<H, T/binary>>, X) when $0 =< H, H =< $9 ->
-    bin_to_int(T, (X*10)+(H-$0));
-bin_to_int(Bin, X) ->
-    {X, Bin}.
+%% WARNING! For simplicity, this function searches only MongooseIM code dir
+-spec find_behaviour_implementations(Behaviour :: module()) -> [module()].
+find_behaviour_implementations(Behaviour) ->
+    {ok, EbinFiles} = file:list_dir(code:lib_dir(mongooseim, ebin)),
+    Mods = [ list_to_atom(filename:rootname(File))
+             || File <- EbinFiles, filename:extension(File) == ".beam" ],
+    lists:filter(fun(M) ->
+                         try lists:keyfind([Behaviour], 2, M:module_info(attributes)) of
+                             {behavior, _} -> true;
+                             {behaviour, _} -> true;
+                             _ -> false
+                         catch
+                             _:_ -> false
+                         end
+                 end, Mods).
 
+%% ------------------------------------------------------------------
+%% Logging
+%% ------------------------------------------------------------------
 
 %% @doc Database backends for various modules return ok, {atomic, ok}
 %% or {atomic, []} on success, and usually {error, ...} on failure.

src/pubsub/mod_pubsub.erl

@@ -262,14 +262,24 @@ process_packet(Acc, From, To, El, #state{server_host = ServerHost, access = Acce
 get_personal_data(Username, Server) ->
      LUser = jid:nodeprep(Username),
      LServer = jid:nodeprep(Server),
-     Payloads = mod_pubsub_db_backend:get_user_payloads(LUser, LServer),
-     Nodes = mod_pubsub_db_backend:get_user_nodes(LUser, LServer),
-     Subscriptions = mod_pubsub_db_backend:get_user_subscriptions(LUser, LServer),
+     Backends = mongoose_lib:find_behaviour_implementations(mod_pubsub_db),
+     Payloads = get_personal_data_group(LUser, LServer, Backends, get_user_payloads),
+     Nodes = get_personal_data_group(LUser, LServer, Backends, get_user_nodes),
+     Subscriptions = get_personal_data_group(LUser, LServer, Backends, get_user_subscriptions),
 
      [{pubsub_payloads, ["node_name", "item_id", "payload"], Payloads},
       {pubsub_nodes, ["node_name", "type"], Nodes},
       {pubsub_subscriptions, ["node_name"], Subscriptions}].
 
+get_personal_data_group(LUser, LServer, Backends, FunctionName) ->
+    lists:flatmap(fun(B) -> try B:FunctionName(LUser, LServer) of
+                                Result when is_list(Result) -> Result;
+                                _ -> []
+                            catch
+                                _:_ -> []
+                            end
+                  end, Backends).
+
 %%====================================================================
 %% gen_server callbacks
 %%====================================================================