Update IP decap testing for dualtor (sonic-net#3702)

Fixes sonic-net/sonic-buildimage#8098

What is the motivation for this PR?
The IP decap testing was only designed for single ToR. This PR is to adapt the IP decap testing for dualtor testbed as well.

How did you do it?
Changes:
* Add common fib_utils.py script for FIB related helper functions.
* Updated the fib testing to use the tools in common fib_utils.py file.
* Update the IP decap testing scripts to support dualtor.

Signed-off-by: Xin Wang <xiwang5@microsoft.com>

ansible/roles/test/files/ptftests/fib_test.py

@@ -4,7 +4,10 @@
                 Design is available in https://github.com/Azure/SONiC/wiki/FIB-Scale-Test-Plan
 
 Usage:          Examples of how to use log analyzer
-                ptf --test-dir ptftests fib_test.FibTest --platform-dir ptftests --qlen=2000 --platform remote -t 'setup_info="/root/test_fib_setup_info.json";testbed_mtu=1514;ipv4=True;test_balancing=True;ipv6=True' --relax --debug info --log-file /tmp/fib_test.FibTest.ipv4.True.ipv6.True.2020-12-22-08:17:05.log --socket-recv-size 16384
+                ptf --test-dir ptftests fib_test.FibTest --platform-dir ptftests --qlen=2000 --platform remote \
+                    -t 'setup_info="/root/test_fib_setup_info.json";testbed_mtu=1514;ipv4=True;test_balancing=True;\
+                    ipv6=True' --relax --debug info --socket-recv-size 16384 \
+                    --log-file /tmp/fib_test.FibTest.ipv4.True.ipv6.True.2020-12-22-08:17:05.log
 '''
 
 #---------------------------------------------------------------------
@@ -167,6 +170,7 @@ def get_src_and_exp_ports(self, dst_ip):
             exp_port_list = next_hop.get_next_hop_list()
             if src_port in exp_port_list:
                 continue
+            logging.info('src_port={}, exp_port_list={}, active_dut_index={}'.format(src_port, exp_port_list, active_dut_index))
             break
         return src_port, exp_port_list, next_hop
 
@@ -293,7 +297,7 @@ def check_ipv4_route(self, src_port, dst_ip_addr, dst_port_list):
                     dport))
 
         if self.pkt_action == self.ACTION_FWD:
-            rcvd_port, rcvd_pkt = verify_packet_any_port(self,masked_exp_pkt,dst_port_list)
+            rcvd_port, rcvd_pkt = verify_packet_any_port(self,masked_exp_pkt, dst_port_list)
             exp_src_mac = self.router_macs[self.ptf_test_port_map[str(dst_port_list[rcvd_port])]['target_dut']]
             actual_src_mac = Ether(rcvd_pkt).src
             if exp_src_mac != actual_src_mac:

tests/common/fixtures/fib_utils.py

@@ -0,0 +1,216 @@
+import json
+import logging
+import tempfile
+
+from datetime import datetime
+
+import pytest
+
+logger = logging.getLogger(__name__)
+
+
+def get_t2_fib_info(duthosts, duts_cfg_facts, duts_mg_facts):
+    """Get parsed FIB information from redis DB for T2 topology.
+
+    Args:
+        duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts
+        duts_cfg_facts (dict): Running config facts of all DUT hosts.
+        duts_mg_facts (dict): Minigraph facts of all DUT hosts.
+
+    Returns:
+        dict: Map of prefix to PTF ports that are connected to DUT output ports.
+            {
+                '192.168.0.0/21': [],
+                '192.168.8.0/25': [[58 59] [62 63] [66 67] [70 71]],
+                '192.168.16.0/25': [[58 59] [62 63] [66 67] [70 71]],
+                ...
+                '20c0:c2e8:0:80::/64': [[58 59] [62 63] [66 67] [70 71]],
+                '20c1:998::/64': [[58 59] [62 63] [66 67] [70 71]],
+                ...
+            }
+    """
+    timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S')
+    fib_info = {}
+    for duthost in duthosts.frontend_nodes:
+        cfg_facts = duts_cfg_facts[duthost.hostname]
+        mg_facts = duts_mg_facts[duthost.hostname]
+        for asic_index, asic_cfg_facts in  enumerate(cfg_facts):
+            asic = duthost.asic_instance(asic_index)
+
+            asic.shell("{} redis-dump -d 0 -k 'ROUTE*' -y > /tmp/fib.{}.txt".format(asic.ns_arg, timestamp))
+            duthost.fetch(src="/tmp/fib.{}.txt".format(timestamp), dest="/tmp/fib")
+
+            po = asic_cfg_facts.get('PORTCHANNEL', {})
+            ports = asic_cfg_facts.get('PORT', {})
+
+            with open("/tmp/fib/{}/tmp/fib.{}.txt".format(duthost.hostname, timestamp)) as fp:
+                fib = json.load(fp)
+                for k, v in fib.items():
+                    skip = False
+
+                    prefix = k.split(':', 1)[1]
+                    ifnames = v['value']['ifname'].split(',')
+                    nh = v['value']['nexthop']
+
+                    oports = []
+                    for ifname in ifnames:
+                        if po.has_key(ifname):
+                            # ignore the prefix, if the prefix nexthop is not a frontend port
+                            if 'members' in po[ifname]:
+                                if 'role' in ports[po[ifname]['members'][0]] and ports[po[ifname]['members'][0]]['role'] == 'Int':
+                                    skip = True
+                                else:
+                                    oports.append([str(mg_facts['minigraph_ptf_indices'][x]) for x in po[ifname]['members']])
+                        else:
+                            if ports.has_key(ifname):
+                                if 'role' in ports[ifname] and ports[ifname]['role'] == 'Int':
+                                    skip = True
+                                else:
+                                    oports.append([str(mg_facts['minigraph_ptf_indices'][ifname])])
+                            else:
+                                logger.info("Route point to non front panel port {}:{}".format(k, v))
+                                skip = True
+
+                    # skip direct attached subnet
+                    if nh == '0.0.0.0' or nh == '::' or nh == "":
+                        skip = True
+
+                    if not skip:
+                        if prefix in fib_info:
+                            fib_info[prefix] += oports
+                        else:
+                            fib_info[prefix] = oports
+
+    return fib_info
+
+
+def get_fib_info(duthost, dut_cfg_facts, duts_mg_facts):
+    """Get parsed FIB information from redis DB.
+
+    Args:
+        duthost (SonicHost): Object for interacting with DUT.
+        duts_cfg_facts (dict): Running config facts of all DUT hosts.
+        duts_mg_facts (dict): Minigraph facts of all DUT hosts.
+
+    Returns:
+        dict: Map of prefix to PTF ports that are connected to DUT output ports.
+            {
+                '192.168.0.0/21': [],
+                '192.168.8.0/25': [[58 59] [62 63] [66 67] [70 71]],
+                '192.168.16.0/25': [[58 59] [62 63] [66 67] [70 71]],
+                ...
+                '20c0:c2e8:0:80::/64': [[58 59] [62 63] [66 67] [70 71]],
+                '20c1:998::/64': [[58 59] [62 63] [66 67] [70 71]],
+                ...
+            }
+    """
+    timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S')
+    fib_info = {}
+    for asic_index, asic_cfg_facts in enumerate(dut_cfg_facts):
+
+        asic = duthost.asic_instance(asic_index)
+
+        asic.shell("{} redis-dump -d 0 -k 'ROUTE*' -y > /tmp/fib.{}.txt".format(asic.ns_arg, timestamp))
+        duthost.fetch(src="/tmp/fib.{}.txt".format(timestamp), dest="/tmp/fib")
+
+        po = asic_cfg_facts.get('PORTCHANNEL', {})
+        ports = asic_cfg_facts.get('PORT', {})
+
+        with open("/tmp/fib/{}/tmp/fib.{}.txt".format(duthost.hostname, timestamp)) as fp:
+            fib = json.load(fp)
+            for k, v in fib.items():
+                skip = False
+
+                prefix = k.split(':', 1)[1]
+                ifnames = v['value']['ifname'].split(',')
+                nh = v['value']['nexthop']
+
+                oports = []
+                for ifname in ifnames:
+                    if po.has_key(ifname):
+                        # ignore the prefix, if the prefix nexthop is not a frontend port
+                        if 'members' in po[ifname]:
+                            if 'role' in ports[po[ifname]['members'][0]] and ports[po[ifname]['members'][0]]['role'] == 'Int':
+                                skip = True
+                            else:
+                                oports.append([str(duts_mg_facts['minigraph_ptf_indices'][x]) for x in po[ifname]['members']])
+                    else:
+                        if ports.has_key(ifname):
+                            if 'role' in ports[ifname] and ports[ifname]['role'] == 'Int':
+                                skip = True
+                            else:
+                                oports.append([str(duts_mg_facts['minigraph_ptf_indices'][ifname])])
+                        else:
+                            logger.info("Route point to non front panel port {}:{}".format(k, v))
+                            skip = True
+
+                # skip direct attached subnet
+                if nh == '0.0.0.0' or nh == '::' or nh == "":
+                    skip = True
+
+                if not skip:
+                    if prefix in fib_info:
+                        fib_info[prefix] += oports
+                    else:
+                        fib_info[prefix] = oports
+                # For single_asic device, add empty list for directly connected subnets
+                elif skip and not duthost.is_multi_asic:
+                    fib_info[prefix] = []
+
+    return fib_info
+
+
+def gen_fib_info_file(ptfhost, fib_info, filename):
+    """Store FIB info dumped & parsed from database to temporary file, then copy the file to PTF host.
+
+    Args:
+        ptfhost (PTFHost): Instance of PTFHost for interacting with the PTF host.
+        fib_info (dict): FIB info dumped and parsed from database.
+        filename (str): Name of the target FIB info file on PTF host.
+    """
+    tmp_fib_info = tempfile.NamedTemporaryFile()
+    for prefix, oports in fib_info.items():
+        tmp_fib_info.write(prefix)
+        if oports:
+            for op in oports:
+                tmp_fib_info.write(' [{}]'.format(' '.join(op)))
+        else:
+            tmp_fib_info.write(' []')
+        tmp_fib_info.write('\n')
+    tmp_fib_info.flush()
+    ptfhost.copy(src=tmp_fib_info.name, dest=filename)
+
+
+@pytest.fixture(scope='module')
+def fib_info_files(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, tbinfo):
+    """Get FIB info from database and store to text files on PTF host.
+
+    For T2 topology, generate a single file to /root/fib_info_all_duts.txt to PTF host.
+    For other topologies, generate one file for each duthost. File name pattern:
+        /root/fib_info_dut<dut_index>.txt
+
+    Args:
+        duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts.
+        ptfhost (PTFHost): Instance of PTFHost for interacting with the PTF host.
+        duts_running_config_facts (dict): Running config facts of all DUT hosts.
+        duts_minigraph_facts (dict): Minigraph facts of all DUT hosts.
+        tbinfo (object): Instance of TestbedInfo.
+
+    Returns:
+        list: List of FIB info file names on PTF host.
+    """
+    duts_config_facts = duts_running_config_facts
+    files = []
+    if tbinfo['topo']['type'] != "t2":
+        for dut_index, duthost in enumerate(duthosts):
+            fib_info = get_fib_info(duthost, duts_config_facts[duthost.hostname], duts_minigraph_facts[duthost.hostname])
+            filename = '/root/fib_info_dut{}.txt'.format(dut_index)
+            gen_fib_info_file(ptfhost, fib_info, filename)
+            files.append(filename)
+    else:
+        fib_info = get_t2_fib_info(duthosts, duts_config_facts, duts_minigraph_facts)
+        filename = '/root/fib_info_all_duts.txt'
+        gen_fib_info_file(ptfhost, fib_info, filename)
+        files.append(filename)
+
+    return files

tests/common/fixtures/ptfhost_utils.py

@@ -4,12 +4,14 @@
 import logging
 import yaml
 
+import requests
+
 from ipaddress import ip_interface
 from jinja2 import Template
 from natsort import natsorted
 
 from tests.common import constants
-
+from tests.common.helpers.assertions import pytest_assert as pt_assert
 
 logger = logging.getLogger(__name__)
 
@@ -27,6 +29,7 @@
 ICMP_RESPONDER_CONF_TEMPL = "icmp_responder.conf.j2"
 GARP_SERVICE_PY = 'garp_service.py'
 GARP_SERVICE_CONF_TEMPL = 'garp_service.conf.j2'
+PTF_TEST_PORT_MAP = '/root/ptf_test_port_map.json'
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -267,3 +270,52 @@ def run_garp_service(duthost, ptfhost, tbinfo, change_mac_addresses, request):
     if tbinfo['topo']['type'] == 't0':
         logger.info("Stopping GARP service on PTF host")
         ptfhost.shell('supervisorctl stop garp_service')
+
+
+def ptf_test_port_map(ptfhost, tbinfo, duthosts, mux_server_url):
+    active_dut_map = {}
+    if 'dualtor' in tbinfo['topo']['name']:
+        res = requests.get(mux_server_url)
+        pt_assert(res.status_code==200, 'Failed to get mux status: {}'.format(res.text))
+        for mux_status in res.json().values():
+            active_dut_index = 0 if mux_status['active_side'] == 'upper_tor' else 1
+            active_dut_map[str(mux_status['port_index'])] = active_dut_index
+
+    disabled_ptf_ports = set()
+    for ptf_map in tbinfo['topo']['ptf_map_disabled'].values():
+        # Loop ptf_map of each DUT. Each ptf_map maps from ptf port index to dut port index
+        disabled_ptf_ports = disabled_ptf_ports.union(set(ptf_map.keys()))
+
+    router_macs = [duthost.facts['router_mac'] for duthost in duthosts]
+
+    logger.info('active_dut_map={}'.format(active_dut_map))
+    logger.info('disabled_ptf_ports={}'.format(disabled_ptf_ports))
+    logger.info('router_macs={}'.format(router_macs))
+
+    ports_map = {}
+    for ptf_port, dut_intf_map in tbinfo['topo']['ptf_dut_intf_map'].items():
+        if str(ptf_port) in disabled_ptf_ports:
+            # Skip PTF ports that are connected to disabled VLAN interfaces
+            continue
+
+        if len(dut_intf_map.keys()) == 2:
+            # PTF port is mapped to two DUTs -> dualtor topology and the PTF port is a vlan port
+            # Packet sent from this ptf port will only be accepted by the active side DUT
+            # DualToR DUTs use same special Vlan interface MAC address
+            target_dut_index = int(active_dut_map[ptf_port])
+            ports_map[ptf_port] = {
+                'target_dut': target_dut_index,
+                'target_mac': tbinfo['topo']['properties']['topology']['DUT']['vlan_configs']['one_vlan_a']['Vlan1000']['mac']
+            }
+        else:
+            # PTF port is mapped to single DUT
+            target_dut_index = int(dut_intf_map.keys()[0])
+            ports_map[ptf_port] = {
+                'target_dut': target_dut_index,
+                'target_mac': router_macs[target_dut_index]
+            }
+
+    logger.debug('ptf_test_port_map={}'.format(json.dumps(ports_map, indent=2)))
+
+    ptfhost.copy(content=json.dumps(ports_map), dest=PTF_TEST_PORT_MAP)
+    return PTF_TEST_PORT_MAP

tests/conftest.py

@@ -1067,3 +1067,45 @@ def cleanup_cache_for_session(request):
     cache.cleanup(zone=tbname)
     for a_dut in tbinfo['duts']:
         cache.cleanup(zone=a_dut)
+
+
+@pytest.fixture(scope='session')
+def duts_running_config_facts(duthosts):
+    """Return running config facts for all multi-ASIC DUT hosts
+
+    Args:
+        duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts.
+
+    Returns:
+        dict: {
+            <dut hostname>: [
+                {asic0_cfg_facts},
+                {asic1_cfg_facts}
+            ]
+        }
+    """
+    cfg_facts = {}
+    for duthost in duthosts:
+        cfg_facts[duthost.hostname] = []
+        for asic in duthost.asics:
+            if asic.is_it_backend():
+                continue
+            asic_cfg_facts = asic.config_facts(source='running')['ansible_facts']
+            cfg_facts[duthost.hostname].append(asic_cfg_facts)
+    return cfg_facts
+
+
+@pytest.fixture(scope='module')
+def duts_minigraph_facts(duthosts, tbinfo):
+    """Return minigraph facts for all DUT hosts
+
+    Args:
+        duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts.
+        tbinfo (object): Instance of TestbedInfo.
+
+    Returns:
+        dict: {
+            <dut hostname>: {dut_minigraph_facts}
+        }
+    """
+    return duthosts.get_extended_minigraph_facts(tbinfo)