CT: test UPF reselection

* support multiple active UPF in test suites
* test case for UPF reselection and invalid pools

.travis.yml

@@ -31,6 +31,7 @@ before_script:
       sudo ip addr add fd96:dcd2:efdb:41c3::20/64 dev lo;
       sudo ip addr add fd96:dcd2:efdb:41c3::30/64 dev lo;
       sudo ip addr add fd96:dcd2:efdb:41c3::40/64 dev lo;
+      sudo ip addr add fd96:dcd2:efdb:41c3::50/64 dev lo;
     fi
 
 script:

test/bench_SUITE.erl

@@ -344,7 +344,7 @@
 	 {[node_selection, {default, 2}, 2, "topon.s5s8.pgw.$ORIGIN"],
 	  {fun node_sel_update/2, final_gsn}},
 	 {[node_selection, {default, 2}, 2, "topon.sx.prox01.$ORIGIN"],
-	  {fun node_sel_update/2, pgw_u_sx}}
+	  {fun node_sel_update/2, pgw_u01_sx}}
 	]).
 
 node_sel_update(Node, {_,_,_,_} = IP) ->
@@ -377,7 +377,7 @@ bench_init_per_group(Config0) ->
     [application:load(App) || App <- [cowboy, ergw, ergw_aaa]],
     load_config(AppCfg),
     {ok, _} = application:ensure_all_started(ergw),
-    {ok, _} = ergw_test_sx_up:start('pgw-u', proplists:get_value(pgw_u_sx, Config)),
+    {ok, _} = ergw_test_sx_up:start('pgw-u01', proplists:get_value(pgw_u01_sx, Config)),
     {ok, _} = ergw_test_sx_up:start('sgw-u', proplists:get_value(sgw_u_sx, Config)),
     {ok, AppsCfg} = application:get_env(ergw_aaa, apps),
     [{aaa_cfg, AppsCfg} |Config].
@@ -395,7 +395,7 @@ init_per_group(ipv4, Config0) ->
     bench_init_per_group(Config).
 
 bench_end_per_group(Config) ->
-    ok = ergw_test_sx_up:stop('pgw-u'),
+    ok = ergw_test_sx_up:stop('pgw-u01'),
     ok = ergw_test_sx_up:stop('sgw-u'),
     ?config(table_owner, Config) ! stop,
     [application:stop(App) || App <- [ranch, cowboy, ergw, ergw_aaa]],
@@ -421,8 +421,8 @@ all() ->
 %%%===================================================================
 
 init_per_testcase(Config) ->
-    ergw_test_sx_up:reset('pgw-u'),
-    ergw_test_sx_up:history('pgw-u', false),
+    ergw_test_sx_up:reset('pgw-u01'),
+    ergw_test_sx_up:history('pgw-u01', false),
     start_gtpc_server(Config),
     reconnect_all_sx_nodes(),
     ok.

test/ergw_http_api_SUITE.erl

@@ -21,7 +21,7 @@
 	 {[node_selection, {default, 2}, 2, "topon.gn.ggsn.$ORIGIN"],
 	  {fun node_sel_update/2, final_gsn}},
 	 {[node_selection, {default, 2}, 2, "topon.sx.prox01.$ORIGIN"],
-	  {fun node_sel_update/2, pgw_u_sx}}
+	  {fun node_sel_update/2, pgw_u01_sx}}
 	]).
 
 -define(TEST_CONFIG,
@@ -154,7 +154,7 @@ init_per_testcase(Config) ->
     meck_reset(Config).
 init_per_testcase(http_api_delete_sessions, Config) ->
     ct:pal("Sockets: ~p", [ergw_gtp_socket_reg:all()]),
-    ergw_test_sx_up:reset('pgw-u'),
+    ergw_test_sx_up:reset('pgw-u01'),
     meck_reset(Config),
     start_gtpc_server(Config),
     Config;

test/ergw_test_lib.erl

@@ -79,11 +79,13 @@ lib_init_per_suite(Config0) ->
 
     case proplists:get_value(upf, Config, true) of
 	true ->
-	    {ok, _} = ergw_test_sx_up:start('pgw-u', proplists:get_value(pgw_u_sx, Config)),
+	    {ok, _} = ergw_test_sx_up:start('pgw-u01', proplists:get_value(pgw_u01_sx, Config)),
+	    {ok, _} = ergw_test_sx_up:start('pgw-u02', proplists:get_value(pgw_u02_sx, Config)),
 	    {ok, _} = ergw_test_sx_up:start('sgw-u', proplists:get_value(sgw_u_sx, Config)),
 	    {ok, _} = ergw_test_sx_up:start('tdf-u', proplists:get_value(tdf_u_sx, Config));
 	_ ->
-	    ok = ergw_test_sx_up:stop('pgw-u'),
+	    ok = ergw_test_sx_up:stop('pgw-u01'),
+	    ok = ergw_test_sx_up:stop('pgw-u02'),
 	    ok = ergw_test_sx_up:stop('sgw-u'),
 	    ok = ergw_test_sx_up:stop('tdf-u')
     end,
@@ -92,7 +94,8 @@ lib_init_per_suite(Config0) ->
 
 lib_end_per_suite(Config) ->
     meck_unload(Config),
-    ok = ergw_test_sx_up:stop('pgw-u'),
+    ok = ergw_test_sx_up:stop('pgw-u01'),
+    ok = ergw_test_sx_up:stop('pgw-u02'),
     ok = ergw_test_sx_up:stop('sgw-u'),
     ok = ergw_test_sx_up:stop('tdf-u'),
     ?config(table_owner, Config) ! stop,
@@ -130,7 +133,8 @@ group_config(ipv4, Config) ->
 	    {proxy_gsn, ?PROXY_GSN_IPv4},
 	    {final_gsn, ?FINAL_GSN_IPv4},
 	    {sgw_u_sx, ?SGW_U_SX_IPv4},
-	    {pgw_u_sx, ?PGW_U_SX_IPv4},
+	    {pgw_u01_sx, ?PGW_U01_SX_IPv4},
+	    {pgw_u02_sx, ?PGW_U02_SX_IPv4},
 	    {tdf_u_sx, ?TDF_U_SX_IPv4}],
     merge_config(Opts, Config);
 group_config(ipv6, Config) ->
@@ -141,7 +145,8 @@ group_config(ipv6, Config) ->
 	    {proxy_gsn, ?PROXY_GSN_IPv6},
 	    {final_gsn, ?FINAL_GSN_IPv6},
 	    {sgw_u_sx, ?SGW_U_SX_IPv6},
-	    {pgw_u_sx, ?PGW_U_SX_IPv6},
+	    {pgw_u01_sx, ?PGW_U01_SX_IPv6},
+	    {pgw_u02_sx, ?PGW_U02_SX_IPv6},
 	    {tdf_u_sx, ?TDF_U_SX_IPv6}],
     merge_config(Opts, Config).
 
@@ -374,7 +379,10 @@ stop_gtpc_server() ->
     end.
 
 sx_nodes_up(_N, 0) ->
-    ct:fail("Sx nodes failed to come up");
+    Got = ergw_sx_node_reg:available(),
+    Expected = supervisor:which_children(ergw_sx_node_sup),
+    ct:fail("Sx nodes failed to come up~nExpected ~p~nGot ~p",
+	    [Expected, Got]);
 sx_nodes_up(N, Cnt) ->
     case ergw_sx_node_reg:available() of
 	Nodes when map_size(Nodes) =:= N ->

test/ergw_test_lib.hrl

@@ -44,7 +44,8 @@
 -define(FINAL_GSN_IPv4, {127,0,200,1}).
 
 -define(SGW_U_SX_IPv4, {127,0,100,1}).
--define(PGW_U_SX_IPv4, {127,0,200,1}).
+-define(PGW_U01_SX_IPv4, {127,0,200,1}).
+-define(PGW_U02_SX_IPv4, {127,0,200,2}).
 -define(TDF_U_SX_IPv4, {127,0,210,1}).
 
 -define(LOCALHOST_IPv6, {0,0,0,0,0,0,0,1}).
@@ -54,8 +55,9 @@
 -define(FINAL_GSN_IPv6, {16#fd96, 16#dcd2, 16#efdb, 16#41c3, 0, 0, 0, 16#30}).
 
 -define(SGW_U_SX_IPv6, {16#fd96, 16#dcd2, 16#efdb, 16#41c3, 0, 0, 0, 16#20}).
--define(PGW_U_SX_IPv6, {16#fd96, 16#dcd2, 16#efdb, 16#41c3, 0, 0, 0, 16#30}).
--define(TDF_U_SX_IPv6, {16#fd96, 16#dcd2, 16#efdb, 16#41c3, 0, 0, 0, 16#40}).
+-define(PGW_U01_SX_IPv6, {16#fd96, 16#dcd2, 16#efdb, 16#41c3, 0, 0, 0, 16#30}).
+-define(PGW_U02_SX_IPv6, {16#fd96, 16#dcd2, 16#efdb, 16#41c3, 0, 0, 0, 16#40}).
+-define(TDF_U_SX_IPv6, {16#fd96, 16#dcd2, 16#efdb, 16#41c3, 0, 0, 0, 16#50}).
 
 -define('APN-EXAMPLE', [<<"example">>, <<"net">>]).
 -define('APN-ExAmPlE', [<<"eXaMpLe">>, <<"net">>]).

test/ergw_test_sx_up.erl

@@ -34,8 +34,10 @@
 %%% API
 %%%===================================================================
 
-start(Role, IP) ->
-    gen_server:start({local, server_name(Role)}, ?MODULE, [IP], []).
+start(Role, IP) when is_tuple(IP) ->
+    gen_server:start({local, server_name(Role)}, ?MODULE, [IP], []);
+start(_, undefined) ->
+    {ok, ignored}.
 
 stop(Role) ->
     try
@@ -149,9 +151,10 @@ handle_call({send, SEID, #pfcp{} = Msg}, _From, State0) ->
     {reply, ok, State};
 
 handle_call({send, Msg}, _From,
-	    #state{gtp = GtpSocket, cp_ip = IP} = State)
+	    #state{gtp = GtpSocket, cp_ip = IP, up_ip = UpIP} = State)
   when is_binary(Msg) ->
-    [[SxTEI, _SxPid]] = ets:match(gtp_context_reg, {{'cp-socket',{teid,'gtp-u','$1'}},'$2'}),
+    {ok, SxPid} = ergw_sx_node_reg:lookup(ergw_inet:bin2ip(UpIP)),
+    [[SxTEI]] = ets:match(gtp_context_reg, {{'cp-socket',{teid,'gtp-u','$1'}},{'_',SxPid}}),
     BinMsg = gtp_packet:encode(#gtp{version = v1, type = g_pdu, tei = SxTEI, ie = Msg}),
     ok = gen_udp:send(GtpSocket, IP, ?GTP1u_PORT, BinMsg),
     {reply, ok, State};

test/ggsn_SUITE.erl

@@ -348,7 +348,7 @@
 	 {[node_selection, {default, 2}, 2, "topon.gn.ggsn.$ORIGIN"],
 	  {fun node_sel_update/2, final_gsn}},
 	 {[node_selection, {default, 2}, 2, "topon.sx.prox01.$ORIGIN"],
-	  {fun node_sel_update/2, pgw_u_sx}},
+	  {fun node_sel_update/2, pgw_u01_sx}},
 	 {[node_selection, {default, 2}, 2, "topon.sx.prox02.$ORIGIN"],
 	  {fun node_sel_update/2, sgw_u_sx}}
 	]).
@@ -455,11 +455,11 @@ setup_per_testcase(Config) ->
 
 setup_per_testcase(Config, ClearSxHist) ->
     ct:pal("Sockets: ~p", [ergw_gtp_socket_reg:all()]),
-    ergw_test_sx_up:reset('pgw-u'),
+    ergw_test_sx_up:reset('pgw-u01'),
     meck_reset(Config),
     start_gtpc_server(Config),
     reconnect_all_sx_nodes(),
-    ClearSxHist andalso ergw_test_sx_up:history('pgw-u', true),
+    ClearSxHist andalso ergw_test_sx_up:history('pgw-u01', true),
     ok.
 
 init_per_testcase(create_pdp_context_request_aaa_reject, Config) ->
@@ -951,7 +951,7 @@ error_indication() ->
 error_indication(Config) ->
     {GtpC, _, _} = create_pdp_context(Config),
 
-    ergw_test_sx_up:send('pgw-u', make_error_indication_report(GtpC)),
+    ergw_test_sx_up:send('pgw-u01', make_error_indication_report(GtpC)),
 
     ct:sleep(100),
     delete_pdp_context(not_found, GtpC),
@@ -1146,7 +1146,7 @@ update_pdp_context_request_tei_update(Config) ->
     [SMR0|_] = lists:filter(
 		 fun(#pfcp{type = session_modification_request}) -> true;
 		    (_) -> false
-		 end, ergw_test_sx_up:history('pgw-u')),
+		 end, ergw_test_sx_up:history('pgw-u01')),
     SMR = pfcp_packet:to_map(SMR0),
     #{update_far :=
 	  #update_far{
@@ -1448,14 +1448,14 @@ session_accounting(Config) ->
     #{context := Context, pfcp:= PCtx} = gtp_context:info(Pid),
 
     %% make sure we handle that the Sx node is not returning any accounting
-    ergw_test_sx_up:accounting('pgw-u', off),
+    ergw_test_sx_up:accounting('pgw-u01', off),
 
     SessionOpts1 = ergw_test_lib:query_usage_report(Context, PCtx),
     ?equal(false, maps:is_key('InPackets', SessionOpts1)),
     ?equal(false, maps:is_key('InOctets', SessionOpts1)),
 
     %% enable accouting again....
-    ergw_test_sx_up:accounting('pgw-u', on),
+    ergw_test_sx_up:accounting('pgw-u01', on),
 
     SessionOpts2 = ergw_test_lib:query_usage_report(Context, PCtx),
     ?match(#{'InPackets' := 3, 'OutPackets' := 1,
@@ -1540,7 +1540,7 @@ simple_aaa(Config) ->
     [SER|_] = lists:filter(
 		fun(#pfcp{type = session_establishment_request}) -> true;
 		   (_) ->false
-		end, ergw_test_sx_up:history('pgw-u')),
+		end, ergw_test_sx_up:history('pgw-u01')),
 
     URR = lists:sort(maps:get(create_urr, SER#pfcp.ie)),
     ?match(
@@ -1571,7 +1571,7 @@ simple_aaa(Config) ->
 	[#usage_report_trigger{perio = 1},
 	 #volume_measurement{total = 5, uplink = 2, downlink = 3},
 	 #tp_packet_measurement{total = 12, uplink = 5, downlink = 7}],
-    ergw_test_sx_up:usage_report('pgw-u', PCtx, MatchSpec, Report),
+    ergw_test_sx_up:usage_report('pgw-u01', PCtx, MatchSpec, Report),
 
     ct:sleep(100),
     delete_pdp_context(GtpC),
@@ -1638,7 +1638,7 @@ simple_ofcs(Config) ->
     [SER|_] = lists:filter(
 		fun(#pfcp{type = session_establishment_request}) -> true;
 		   (_) ->false
-		end, ergw_test_sx_up:history('pgw-u')),
+		end, ergw_test_sx_up:history('pgw-u01')),
 
     URR = maps:get(create_urr, SER#pfcp.ie),
     ?match(
@@ -1660,7 +1660,7 @@ simple_ofcs(Config) ->
 	[#usage_report_trigger{perio = 1},
 	 #volume_measurement{total = 5, uplink = 2, downlink = 3},
 	 #tp_packet_measurement{total = 12, uplink = 5, downlink = 7}],
-    ergw_test_sx_up:usage_report('pgw-u', PCtx, MatchSpec, Report),
+    ergw_test_sx_up:usage_report('pgw-u01', PCtx, MatchSpec, Report),
 
     ct:sleep(100),
     delete_pdp_context(GtpC),
@@ -1714,7 +1714,7 @@ simple_ocs(Config) ->
     [SER|_] = lists:filter(
 		fun(#pfcp{type = session_establishment_request}) -> true;
 		   (_) ->false
-		end, ergw_test_sx_up:history('pgw-u')),
+		end, ergw_test_sx_up:history('pgw-u01')),
 
     URR = lists:sort(maps:get(create_urr, SER#pfcp.ie)),
     ?match(
@@ -1753,7 +1753,7 @@ simple_ocs(Config) ->
 	[#usage_report_trigger{volqu = 1},
 	 #volume_measurement{total = 5, uplink = 2, downlink = 3},
 	 #tp_packet_measurement{total = 12, uplink = 5, downlink = 7}],
-    ergw_test_sx_up:usage_report('pgw-u', PCtx, MatchSpec, Report),
+    ergw_test_sx_up:usage_report('pgw-u01', PCtx, MatchSpec, Report),
 
     ct:sleep(100),
     delete_pdp_context(GtpC),
@@ -1948,8 +1948,8 @@ volume_threshold(Config) ->
 
     MatchSpec = ets:fun2ms(fun({Id, {'online', _}}) -> Id end),
 
-    ergw_test_sx_up:usage_report('pgw-u', PCtx, MatchSpec, [#usage_report_trigger{volth = 1}]),
-    ergw_test_sx_up:usage_report('pgw-u', PCtx, MatchSpec, [#usage_report_trigger{volqu = 1}]),
+    ergw_test_sx_up:usage_report('pgw-u01', PCtx, MatchSpec, [#usage_report_trigger{volth = 1}]),
+    ergw_test_sx_up:usage_report('pgw-u01', PCtx, MatchSpec, [#usage_report_trigger{volqu = 1}]),
 
     ct:sleep({seconds, 1}),
 
@@ -1959,7 +1959,7 @@ volume_threshold(Config) ->
 	lists:filter(
 	  fun(#pfcp{type = session_modification_request}) -> true;
 	     (_) ->false
-	  end, ergw_test_sx_up:history('pgw-u')),
+	  end, ergw_test_sx_up:history('pgw-u01')),
 
     ?equal([0, 0, 0, 0, 0, 1, 0, 0, 0],
 	   [maps_key_length(X1, Sx1#pfcp.ie)
@@ -2163,7 +2163,7 @@ gx_rar(Config) ->
 	lists:filter(
 	  fun(#pfcp{type = session_modification_request}) -> true;
 	     (_) ->false
-	  end, ergw_test_sx_up:history('pgw-u')),
+	  end, ergw_test_sx_up:history('pgw-u01')),
 
     ct:pal("Sx1: ~p", [Sx1]),
     ?equal([2, 2, 1, 0, 0, 0, 0, 0, 0],

test/ggsn_proxy_SUITE.erl

@@ -428,7 +428,7 @@
 	 {[node_selection, {default, 2}, 2, "topon.sx.sgw-u01.$ORIGIN"],
 	  {fun node_sel_update/2, sgw_u_sx}},
 	 {[node_selection, {default, 2}, 2, "topon.sx.pgw-u01.$ORIGIN"],
-	  {fun node_sel_update/2, pgw_u_sx}},
+	  {fun node_sel_update/2, pgw_u01_sx}},
 	 {[node_selection, {default, 2}, 2, "topon.pgw-1.nodes.$ORIGIN"],
 	  {fun node_sel_update/2, final_gsn}},
 	 {[node_selection, {default, 2}, 2, "topon.upf-1.nodes.$ORIGIN"],
@@ -445,7 +445,7 @@
 	 {[node_selection, {default, 2}, 2, "topon.sx.sgw-u01.$ORIGIN"],
 	  {fun node_sel_update/2, sgw_u_sx}},
 	 {[node_selection, {default, 2}, 2, "topon.sx.pgw-u01.$ORIGIN"],
-	  {fun node_sel_update/2, pgw_u_sx}},
+	  {fun node_sel_update/2, pgw_u01_sx}},
 	 {[node_selection, {default, 2}, 2, "topon.pgw-1.nodes.$ORIGIN"],
 	  {fun node_sel_update/2, final_gsn}},
 	 {[node_selection, {default, 2}, 2, "topon.upf-1.nodes.$ORIGIN"],
@@ -557,12 +557,12 @@ setup_per_testcase(Config) ->
 
 setup_per_testcase(Config, ClearSxHist) ->
     ct:pal("Sockets: ~p", [ergw_gtp_socket_reg:all()]),
-    ergw_test_sx_up:reset('pgw-u'),
+    ergw_test_sx_up:reset('pgw-u01'),
     ergw_test_sx_up:reset('sgw-u'),
     meck_reset(Config),
     start_gtpc_server(Config),
     reconnect_all_sx_nodes(),
-    ClearSxHist andalso ergw_test_sx_up:history('pgw-u', true),
+    ClearSxHist andalso ergw_test_sx_up:history('pgw-u01', true),
     ok.
 
 init_per_testcase(path_restart, Config) ->