Add dist_util_spawn_options in kernel

This change allows user to specify spawn options when dist protocol starts dist_util:handshake_other_started or handshake_we_started processes. These processes will be alive until the connections are down. With thousands of ssl dist connections it spends quite amount of memory, and it would be great if we can change garbage collection, as well as other spawn options for dist_util processes.
erlang · Aug 16, 2021 · e44b432 · e44b432
1 parent 16d9f17
commit e44b432
Show file tree

Hide file tree

Showing 5 changed files with 118 additions and 4 deletions.
diff --git a/lib/kernel/doc/src/kernel_app.xml b/lib/kernel/doc/src/kernel_app.xml
@@ -293,6 +293,15 @@
 	  other distributed Erlang nodes.
 	  See <seemfa marker="gen_tcp#connect/4"><c>gen_tcp:connect/4</c></seemfa>.</p>
       </item>
+      <tag><c>{net_ticker_spawn_options, Opts}</c></tag>
+      <item>
+        <marker id="net_ticker_spawn_options"></marker>
+        <p>Defines a list of extra spawn options for dist_util:con_loop processes.
+          When there is a large number of dist connections, setting up garbage collection
+          options can be helpful to reduce memory usage. Default is [link, {priority, max}],
+          and these two options cannot be changed.
+          See <seemfa marker="erts:erlang#spawn_opt/2"><c>erlang:spawn_opt/2</c></seemfa>.</p>
+      </item>
       <tag><c>inet_parse_error_log = silent</c></tag>
       <item>
         <p>If set, no log events are issued when erroneous lines are

diff --git a/lib/kernel/src/inet_tcp_dist.erl b/lib/kernel/src/inet_tcp_dist.erl
@@ -208,7 +208,7 @@ accept_connection(AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
 gen_accept_connection(Driver, AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
     spawn_opt(?MODULE, do_accept,
 	      [Driver, self(), AcceptPid, Socket, MyNode, Allowed, SetupTime],
-	      [link, {priority, max}]).
+	      net_ticker_spawn_options()).
 
 do_accept(Driver, Kernel, AcceptPid, Socket, MyNode, Allowed, SetupTime) ->
     receive
@@ -275,6 +275,14 @@ nodelay() ->
     end.
 
 
+% we may want different spawn options for dist_util processes
+
+net_ticker_spawn_options() ->
+    Opts = application:get_env(kernel, net_ticker_spawn_options, []),
+    Opts1 = [{priority, max} | proplists:delete(priority, Opts)],
+    [link | proplists:delete(link, Opts1)].
+
+
 %% ------------------------------------------------------------
 %% Get remote information about a Socket.
 %% ------------------------------------------------------------
@@ -304,7 +312,7 @@ setup(Node, Type, MyNode, LongOrShortNames,SetupTime) ->
 gen_setup(Driver, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
     spawn_opt(?MODULE, do_setup, 
 	      [Driver, self(), Node, Type, MyNode, LongOrShortNames, SetupTime],
-	      [link, {priority, max}]).
+	      net_ticker_spawn_options()).
 
 do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
     ?trace("~p~n",[{inet_tcp_dist,self(),setup,Node}]),

diff --git a/lib/kernel/test/erl_distribution_SUITE.erl b/lib/kernel/test/erl_distribution_SUITE.erl
@@ -34,6 +34,7 @@
 	 setopts/1,
 	 table_waste/1, net_setuptime/1,
 	 inet_dist_options_options/1,
+         net_ticker_spawn_options/1,
 
 	 monitor_nodes_nodedown_reason/1,
 	 monitor_nodes_complex_nodedown_reason/1,
@@ -52,6 +53,7 @@
 
 %% Performs the test at another node.
 -export([get_socket_priorities/0,
+         get_net_ticker_fullsweep_option/1,
 	 tick_cli_test/1, tick_cli_test1/1,
 	 tick_serv_test/2, tick_serv_test1/1,
 	 run_remote_test/1,
@@ -92,6 +94,7 @@ all() ->
      epmd_reconnect,
      hidden_node, setopts,
      table_waste, net_setuptime, inet_dist_options_options,
+     net_ticker_spawn_options,
      {group, monitor_nodes},
      erl_uds_dist_smoke_test,
      erl_1424].
@@ -1060,6 +1063,48 @@ get_socket_priorities() ->
 
 
 
+%% check net_ticker_spawn_options
+net_ticker_spawn_options(Config) when is_list(Config) ->
+    FullsweepString0 = "[{fullsweep_after,0}]",
+    FullsweepString =
+        case os:cmd("echo [{a,1}]") of
+            "[{a,1}]"++_ ->
+                FullsweepString0;
+            _ ->
+                %% Some shells need quoting of [{}]
+                "'"++FullsweepString0++"'"
+        end,
+    InetDistOptions =
+        "-hidden "
+        "-kernel net_ticker_spawn_options "++FullsweepString,
+    {ok,Node1} =
+        start_node("", net_ticker_spawn_options_1, InetDistOptions),
+    {ok,Node2} =
+        start_node("", net_ticker_spawn_options_2, InetDistOptions),
+    %%
+    pong =
+        rpc:call(Node1, net_adm, ping, [Node2]),
+    FullsweepOptionNode1 =
+        rpc:call(Node1, ?MODULE, get_net_ticker_fullsweep_option, [Node2]),
+    FullsweepOptionNode2 =
+        rpc:call(Node2, ?MODULE, get_net_ticker_fullsweep_option, [Node1]),
+    io:format("FullsweepOptionNode1 = ~p", [FullsweepOptionNode1]),
+    io:format("FullsweepOptionNode2 = ~p", [FullsweepOptionNode2]),
+    0 = FullsweepOptionNode1,
+    0 = FullsweepOptionNode2,
+    %%
+    stop_node(Node2),
+    stop_node(Node1),
+    ok.
+
+get_net_ticker_fullsweep_option(Node) ->
+    Port = proplists:get_value(Node, erlang:system_info(dist_ctrl)),
+    {links, [DistUtilPid, _NetKernelPid]} = erlang:port_info(Port, links),
+    {garbage_collection, GCOpts} = erlang:process_info(DistUtilPid, garbage_collection),
+    proplists:get_value(fullsweep_after, GCOpts).
+
+
+
 %%
 %% Testcase:
 %%   monitor_nodes_nodedown_reason

diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl
@@ -448,7 +448,7 @@ gen_accept_connection(
                   Driver, AcceptPid, DistCtrl,
                   MyNode, Allowed, SetupTime, Kernel)
         end,
-        [link, {priority, max}])).
+        net_ticker_spawn_options())).
 
 do_accept(
   _Driver, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime, Kernel) ->
@@ -540,7 +540,7 @@ gen_setup(Driver, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
     Kernel = self(),
     monitor_pid(
       spawn_opt(setup_fun(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime),
-                [link, {priority, max}])).
+                net_ticker_spawn_options())).
 
 -spec setup_fun(_,_,_,_,_,_,_) -> fun(() -> no_return()).
 setup_fun(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
@@ -808,6 +808,13 @@ nodelay() ->
     end.
 
 
+% we may want different spawn options for dist_util processes
+
+net_ticker_spawn_options() ->
+    Opts = application:get_env(kernel, net_ticker_spawn_options, []),
+    Opts1 = [{priority, max} | proplists:delete(priority, Opts)],
+    [link | proplists:delete(link, Opts1)].
+
 get_ssl_options(Type) ->
     try ets:lookup(ssl_dist_opts, Type) of
         [{Type, Opts0}] ->

diff --git a/lib/ssl/test/ssl_dist_SUITE.erl b/lib/ssl/test/ssl_dist_SUITE.erl
@@ -53,6 +53,8 @@
          listen_options/1,
          connect_options/0,
          connect_options/1,
+         net_ticker_spawn_options/0,
+         net_ticker_spawn_options/1,
          use_interface/0,
          use_interface/1,
          verify_fun_fail/0,
@@ -70,6 +72,7 @@
          listen_options_test/3,
          do_connect_options/2,
          connect_options_test/3,
+         net_ticker_spawn_options_test/3,
          verify_fun_fail_test/3,
          verify_fun_pass_test/3,
          verify_pass_always/3,
@@ -100,6 +103,7 @@ all() ->
      listen_port_options,
      listen_options,
      connect_options,
+     net_ticker_spawn_options,
      use_interface,
      verify_fun_fail,
      verify_fun_pass
@@ -322,6 +326,22 @@ connect_options() ->
 connect_options(Config) when is_list(Config) ->
     try_setting_priority(fun do_connect_options/2, Config).
 
+%%--------------------------------------------------------------------
+net_ticker_spawn_options() ->
+    [{doc, "Test net_ticker_spawn_options"}].
+net_ticker_spawn_options(Config) when is_list(Config) ->
+    FullsweepString0 = "[{fullsweep_after,0}]",
+    FullsweepString =
+        case os:cmd("echo [{a,1}]") of
+            "[{a,1}]"++_ ->
+                FullsweepString0;
+            _ ->
+                %% Some shells need quoting of [{}]
+                "'"++FullsweepString0++"'"
+        end,
+    Options = "-kernel net_ticker_spawn_options "++FullsweepString,
+    gen_dist_test(net_ticker_spawn_options_test, [{tls_only_basic_opts, Options} | Config]).
+
 
 %%--------------------------------------------------------------------
 use_interface() ->
@@ -594,6 +614,23 @@ connect_options_test(NH1, NH2, Config) ->
     %% Node 2 will not, since it only applies to outbound connections.
     [] = Elevated2.
 
+net_ticker_spawn_options_test(NH1, NH2, _Config) ->
+    Node1 = NH1#node_handle.nodename,
+    Node2 = NH2#node_handle.nodename,
+
+    pong = apply_on_ssl_node(NH1, fun () -> net_adm:ping(Node2) end),
+
+    FullsweepOptionNode1 =
+        apply_on_ssl_node(NH1, fun () -> get_dist_util_fullsweep_option(Node2) end),
+    FullsweepOptionNode2 =
+        apply_on_ssl_node(NH2, fun () -> get_dist_util_fullsweep_option(Node1) end),
+
+    ct:pal("FullsweepOptionNode1: ~p~n", [FullsweepOptionNode1]),
+    ct:pal("FullsweepOptionNode2: ~p~n", [FullsweepOptionNode2]),
+
+    0 = FullsweepOptionNode1,
+    0 = FullsweepOptionNode2.
+
 
 verify_fun_fail_test(NH1, NH2, _) ->
     Node2 = NH2#node_handle.nodename,
@@ -635,6 +672,14 @@ get_socket_priorities() ->
 	{ok,[{priority,Priority}]} <-
 	    [inet:getopts(Port, [priority]) || Port <- inet_ports()]].
 
+get_dist_util_fullsweep_option(Node) ->
+    SenderPid = proplists:get_value(Node, erlang:system_info(dist_ctrl)),
+    {links, Links1} = erlang:process_info(SenderPid, links),
+    {links, Links2} = erlang:process_info(whereis(net_kernel), links),
+    [DistUtilPid] = [X || X <- Links1, Y <- Links2, X =:= Y],
+    {garbage_collection, GCOpts} = erlang:process_info(DistUtilPid, garbage_collection),
+    proplists:get_value(fullsweep_after, GCOpts).
+
 inet_ports() ->
      [Port || Port <- erlang:ports(),
               element(2, erlang:port_info(Port, name)) =:= "tcp_inet"].