implement path monitoring profiles for outgoing (active) paths

For an active path, the echo request interval is adjusted depending on
wether the path as active contexts (busy), no contexts (idle) or is
down.
For idle and down paths a maximum time to keep their state, monitor
them or try to reconntact them is introduced.

A path is considered active when we have initiated it (e.g. SGSN/SGW or
proxy use).

.gitlab-ci.yml

@@ -42,6 +42,16 @@ build:otp-23.0:
   script:
     - sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'
     - ip addr add fd96:dcd2:efdb:41c3::10/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::11/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::12/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::13/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::14/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::15/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::16/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::17/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::18/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::19/64 dev lo
+    - ip addr add fd96:dcd2:efdb:41c3::1a/64 dev lo
     - ip addr add fd96:dcd2:efdb:41c3::20/64 dev lo
     - ip addr add fd96:dcd2:efdb:41c3::30/64 dev lo
     - ip addr add fd96:dcd2:efdb:41c3::40/64 dev lo

.travis.yml

@@ -28,6 +28,16 @@ before_script:
   - if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
       sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6';
       sudo ip addr add fd96:dcd2:efdb:41c3::10/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::11/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::12/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::13/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::14/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::15/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::16/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::17/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::18/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::19/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::1a/64 dev lo;
       sudo ip addr add fd96:dcd2:efdb:41c3::20/64 dev lo;
       sudo ip addr add fd96:dcd2:efdb:41c3::30/64 dev lo;
       sudo ip addr add fd96:dcd2:efdb:41c3::40/64 dev lo;

src/ergw.erl

@@ -191,7 +191,7 @@ i(memory, socket) ->
     {socket, MemUsage};
 i(memory, path) ->
     MemUsage =
-	lists:foldl(fun({_, Pid}, Mem) ->
+	lists:foldl(fun({_, Pid, _}, Mem) ->
 			    {memory, M} = erlang:process_info(Pid, memory),
 			    Mem + M
 		    end, 0, gtp_path_reg:all()),

src/ergw_api.erl

@@ -18,7 +18,7 @@
 
 peer(all) ->
     Peers = gtp_path_reg:all(),
-    lists:map(fun({_, Pid}) -> gtp_path:info(Pid) end, Peers);
+    lists:map(fun({_, Pid, _}) -> gtp_path:info(Pid) end, Peers);
 peer({_,_,_,_} = IP) ->
     collect_peer_info(gtp_path_reg:all(IP));
 peer({_,_,_,_,_,_,_,_} = IP) ->
@@ -55,9 +55,9 @@ memory(Limit0) ->
 %%%===================================================================
 
 collect_peer_info(Peers) ->
-    lists:map(fun gtp_path:info/1, Peers).
+    lists:map(fun({Pid, _}) -> gtp_path:info(Pid) end, Peers).
 
-collext_path_contexts(Path, Tunnels) ->
+collext_path_contexts({Path, _State}, Tunnels) ->
     lists:foldl(fun(Pid, TunIn) ->
 			collect_contexts(Pid, TunIn)
 		end, Tunnels, gtp_path:all(Path)).

src/ergw_proxy_lib.erl

@@ -12,7 +12,9 @@
 -export([validate_options/3, validate_option/2,
 	 forward_request/3, forward_request/7, forward_request/9,
 	 get_seq_no/3,
-	 select_gw/4, select_proxy_sockets/3]).
+	 select_gw/5,
+	 select_gtp_proxy_sockets/2,
+	 select_sx_proxy_candidate/3]).
 -export([create_forward_session/3,
 	 modify_forward_session/5,
 	 delete_forward_session/4,
@@ -48,16 +50,28 @@ get_seq_no(#context{control_port = GtpPort}, ReqKey, Request) ->
     ReqId = make_request_id(ReqKey, Request),
     ergw_gtp_c_socket:get_seq_no(GtpPort, ReqId).
 
-select_gw(#{imsi := IMSI, gwSelectionAPN := APN}, Services, NodeSelect, Context) ->
+choose_gw([], _NodeSelect, _Port, Context) ->
+    throw(?CTX_ERR(?FATAL, no_resources_available, Context));
+choose_gw(Nodes, NodeSelect, #gtp_port{name = Name} = Port,
+	  #context{version = Version} = Context) ->
+    {Candidate, Next} = ergw_node_selection:snaptr_candidate(Nodes),
+    {Node, IP} = resolve_gw(Candidate, NodeSelect, Context),
+    case gtp_path_reg:state({Name, Version, IP}) of
+	down ->
+	    choose_gw(Next, NodeSelect, Port, Context);
+	State when State =:= undefined; State =:= up ->
+	    {Node, IP}
+    end.
+
+select_gw(#{imsi := IMSI, gwSelectionAPN := APN}, Services, NodeSelect, Port, Context) ->
     FQDN = ergw_node_selection:apn_to_fqdn(APN, IMSI),
     case ergw_node_selection:candidates(FQDN, Services, NodeSelect) of
 	Nodes when is_list(Nodes), length(Nodes) /= 0 ->
-	    {Node, _} = ergw_node_selection:snaptr_candidate(Nodes),
-	    resolve_gw(Node, NodeSelect, Context);
+	    choose_gw(Nodes, NodeSelect, Port, Context);
 	_ ->
 	    throw(?CTX_ERR(?FATAL, system_failure, Context))
     end;
-select_gw(_ProxyInfo, _Services, _NodeSelect, Context) ->
+select_gw(_ProxyInfo, _Services, _NodeSelect, _Port, Context) ->
     throw(?CTX_ERR(?FATAL, system_failure, Context)).
 
 lb(L) when is_list(L) ->
@@ -83,33 +97,38 @@ select_gw_ip(_IP4, IP6) when length(IP6) /= 0 ->
 select_gw_ip(_IP4, _IP6) ->
     undefined.
 
-select_proxy_sockets({GwNode, _}, #{upfSelectionAPN := APN} = ProxyInfo,
-		     #{contexts := Contexts, proxy_ports := ProxyPorts,
-		       node_selection := ProxyNodeSelect}) ->
+select_gtp_proxy_sockets(ProxyInfo, #{contexts := Contexts, proxy_ports := ProxyPorts}) ->
     Context = maps:get(context, ProxyInfo, default),
     Ctx = maps:get(Context, Contexts, #{}),
     if Ctx =:= #{} ->
 	    ?LOG(warning, "proxy context ~p not found, using default", [Context]);
        true -> ok
     end,
     Cntl = maps:get(proxy_sockets, Ctx, ProxyPorts),
-    NodeSelect = maps:get(node_selection, Ctx, ProxyNodeSelect),
+    ergw_gtp_socket_reg:lookup(lb(Cntl)).
 
+select_sx_proxy_candidate({GwNode, _}, #{upfSelectionAPN := APN} = ProxyInfo,
+			  #{contexts := Contexts, node_selection := ProxyNodeSelect}) ->
+    Context = maps:get(context, ProxyInfo, default),
+    Ctx = maps:get(Context, Contexts, #{}),
+    if Ctx =:= #{} ->
+	    ?LOG(warning, "proxy context ~p not found, using default", [Context]);
+       true -> ok
+    end,
+    NodeSelect = maps:get(node_selection, Ctx, ProxyNodeSelect),
     APN_FQDN = ergw_node_selection:apn_to_fqdn(APN),
     Services = [{"x-3gpp-upf", "x-sxa"}],
     Candidates0 = ergw_node_selection:candidates(APN_FQDN, Services, NodeSelect),
     PGWCandidate = [{GwNode, 0, Services, [], []}],
-    Candidates =
-	case ergw_node_selection:topology_match(Candidates0, PGWCandidate) of
-	    {_, C} when is_list(C), length(C) /= 0 ->
-		C;
-	    {C, _} when is_list(C), length(C) /= 0 ->
-		C;
-	    _ ->
-		%% neither colocation, not topology matched
-		Candidates0
-	end,
-    {ergw_gtp_socket_reg:lookup(lb(Cntl)), Candidates}.
+    case ergw_node_selection:topology_match(Candidates0, PGWCandidate) of
+	{_, C} when is_list(C), length(C) /= 0 ->
+	    C;
+	{C, _} when is_list(C), length(C) /= 0 ->
+	    C;
+	_ ->
+	    %% neither colocation, not topology matched
+	    Candidates0
+    end.
 
 %%%===================================================================
 %%% Options Validation

src/ggsn_gn.erl

@@ -338,7 +338,7 @@ handle_request(ReqKey,
     DAF = proplists:get_bool('Dual Address Bearer Flag', gtp_v1_c:get_common_flags(IEs)),
 
     Context1 = update_context_from_gtp_req(Request, Context0),
-    ContextPreAuth = gtp_path:bind(Request, Context1),
+    ContextPreAuth = gtp_path:bind(Request, false, Context1),
 
     gtp_context:terminate_colliding_context(ContextPreAuth),
 
@@ -425,7 +425,7 @@ handle_request(ReqKey,
 		 'Session' := Session} = Data0) ->
 
     Context0 = update_context_from_gtp_req(Request, OldContext),
-    Context = gtp_path:bind(Request, Context0),
+    Context = gtp_path:bind(Request, false, Context0),
     URRActions = update_session_from_gtp_req(IEs, Session, Context),
 
     Data1 = if Context /= OldContext ->
@@ -489,7 +489,7 @@ handle_response({From, TermCause},
 		     ie = #{?'Cause' := #cause{value = Cause}}} = Response,
 		_Request, _State,
 		#{context := Context0} = Data) ->
-    Context = gtp_path:bind(Response, Context0),
+    Context = gtp_path:bind(Response, false, Context0),
     close_pdp_context(TermCause, Data),
     if is_tuple(From) -> gen_statem:reply(From, {ok, Cause});
        true -> ok
@@ -674,7 +674,7 @@ defer_usage_report(URRActions, UsageReport) ->
 
 apply_context_change(NewContext0, OldContext, URRActions,
 		     #{pfcp := PCtx0, pcc := PCC} = Data) ->
-    NewContext = gtp_path:bind(NewContext0),
+    NewContext = gtp_path:bind(false, NewContext0),
     {PCtx, UsageReport} =
 	ergw_gsn_lib:modify_sgi_session(PCC, URRActions,
 					#{}, NewContext, PCtx0),

src/ggsn_gn_proxy.erl

@@ -276,7 +276,7 @@ handle_request(ReqKey,
 		 'Session' := Session} = Data) ->
 
     Context1 = update_context_from_gtp_req(Request, Context0#context{state = #context_state{}}),
-    Context2 = gtp_path:bind(Request, Context1),
+    Context2 = gtp_path:bind(Request, false, Context1),
 
     gtp_context:terminate_colliding_context(Context2),
     gtp_context:remote_context_register_new(Context2),
@@ -285,20 +285,21 @@ handle_request(ReqKey,
     SessionOpts = ggsn_gn:init_session_from_gtp_req(IEs, AAAopts, SessionOpts0),
 
     ProxyInfo = handle_proxy_info(Request, SessionOpts, Context2, Data),
+    ProxyGtpPort = ergw_proxy_lib:select_gtp_proxy_sockets(ProxyInfo, Data),
 
     %% GTP v1 services only, we don't do v1 to v2 conversion (yet)
     Services = [{"x-3gpp-ggsn", "x-gn"}, {"x-3gpp-ggsn", "x-gp"},
 		{"x-3gpp-pgw", "x-gn"}, {"x-3gpp-pgw", "x-gp"}],
-    ProxyGGSN = ergw_proxy_lib:select_gw(ProxyInfo, Services, NodeSelect, Context2),
+    ProxyGGSN = ergw_proxy_lib:select_gw(ProxyInfo, Services, NodeSelect, ProxyGtpPort, Context2),
+
+    DPCandidates = ergw_proxy_lib:select_sx_proxy_candidate(ProxyGGSN, ProxyInfo, Data),
 
-    {ProxyGtpPort, DPCandidates} =
-	ergw_proxy_lib:select_proxy_sockets(ProxyGGSN, ProxyInfo, Data),
     SxConnectId = ergw_sx_node:request_connect(DPCandidates, NodeSelect, 1000),
 
     {ok, _} = ergw_aaa_session:invoke(Session, SessionOpts, start, #{async => true}),
 
     ProxyContext0 = init_proxy_context(ProxyGtpPort, Context2, ProxyInfo, ProxyGGSN),
-    ProxyContext1 = gtp_path:bind(ProxyContext0),
+    ProxyContext1 = gtp_path:bind(true, ProxyContext0),
 
     ergw_sx_node:wait_connect(SxConnectId),
     {Context, ProxyContext, PCtx} =
@@ -317,7 +318,7 @@ handle_request(ReqKey,
   when ?IS_REQUEST_CONTEXT(ReqKey, Request, OldContext) ->
 
     Context0 = update_context_from_gtp_req(Request, OldContext),
-    Context1 = gtp_path:bind(Request, Context0),
+    Context1 = gtp_path:bind(Request, false, Context0),
 
     gtp_context:remote_context_update(OldContext, Context1),
 
@@ -341,8 +342,8 @@ handle_request(ReqKey,
 		 proxy_context := ProxyContext0} = Data)
   when ?IS_REQUEST_CONTEXT(ReqKey, Request, ProxyContext0) ->
 
-    Context = gtp_path:bind(Context0),
-    ProxyContext = gtp_path:bind(Request, ProxyContext0),
+    Context = gtp_path:bind(false, Context0),
+    ProxyContext = gtp_path:bind(Request, true, ProxyContext0),
 
     DataNew = Data#{context => Context, proxy_context => ProxyContext},
     forward_request(ggsn2sgsn, ReqKey, Request, DataNew, Data),
@@ -355,8 +356,8 @@ handle_request(ReqKey,
 	       #{context := Context0,
 		 proxy_context := ProxyContext0} = Data)
   when ?IS_REQUEST_CONTEXT_OPTIONAL_TEI(ReqKey, Request, Context0) ->
-    Context = gtp_path:bind(Request, Context0),
-    ProxyContext = gtp_path:bind(ProxyContext0),
+    Context = gtp_path:bind(Request, false, Context0),
+    ProxyContext = gtp_path:bind(true, ProxyContext0),
 
     DataNew = Data#{context => Context, proxy_context => ProxyContext},
     forward_request(sgsn2ggsn, ReqKey, Request, DataNew, Data),
@@ -404,7 +405,7 @@ handle_response(#proxy_request{direction = sgsn2ggsn} = ProxyRequest,
     ?LOG(warning, "OK Proxy Response ~p", [Response]),
 
     ProxyContext1 = update_context_from_gtp_req(Response, PrevProxyCtx),
-    ProxyContext = gtp_path:bind(Response, ProxyContext1),
+    ProxyContext = gtp_path:bind(Response, true, ProxyContext1),
     gtp_context:remote_context_register(ProxyContext),
 
     Return =
@@ -532,7 +533,7 @@ handle_sgsn_change(_, _, ProxyContext) ->
 
 update_path_bind(NewContext0, OldContext)
   when NewContext0 /= OldContext ->
-    NewContext = gtp_path:bind(NewContext0),
+    NewContext = gtp_path:bind(false, NewContext0),
     gtp_path:unbind(OldContext),
     NewContext;
 update_path_bind(NewContext, _OldContext) ->