changes to support fabric asic in gen-mg and chassis TestbedProcessing

1. Support chassis, multi-duts scenarios in TestbedProcessing.
When testbed.yaml file contains cardtype=Linecard or cardtype=supervisor,
TestbedProcessing will add that to lab, veos inventory files.
When dut is multi-dut, TestbedProcessing will convert the dut type list to string.
2. Support fabric_info generation in fabric_info.py when the num_asic > 0 and cardtype is supervisor.
3. Use the fabric_info in minigraph templates to generated the fabric asic info
4. Use variable name cardtype instead of type in ansible, dut_utils.py and minigraph templates to be more explicit
5. allow t2 as a topo in testbed.py

ansible/TestbedProcessing.py

@@ -164,7 +164,7 @@ def makeVMHostCreds(data, outfile):
 error handling: checks if attribute values are None type or string "None"
 """
 def makeSonicLabDevices(data, outfile):
-    csv_columns = "Hostname,ManagementIp,HwSku,Type"
+    csv_columns = "Hostname,ManagementIp,HwSku,Type,CardType"
     topology = data
     csv_file = outfile
 
@@ -175,17 +175,19 @@ def makeSonicLabDevices(data, outfile):
                 hostname = device
                 managementIP = str(deviceDetails.get("ansible").get("ansible_host"))
                 hwsku = deviceDetails.get("hwsku")
-                devType = deviceDetails.get("device_type")
-
+                devType = deviceDetails.get("device_type") #DevSonic, server, FanoutRoot etc
+                cardType = deviceDetails.get("cardtype") #supervisor, Linecard etc
                 # catch empty values
                 if not managementIP:
                     managementIP = ""
                 if not hwsku:
                     hwsku = ""
                 if not devType:
                     devType = ""
+                if not cardType:
+                    cardType = ""
 
-                row = hostname + "," + managementIP + "," + hwsku + "," + devType
+                row = hostname + "," + managementIP + "," + hwsku + "," + devType + "," + cardType
                 f.write(row + "\n")
     except IOError:
         print("I/O error: makeSonicLabDevices")
@@ -240,6 +242,11 @@ def makeTestbed(data, outfile):
                     ptf = ""
                 if not comment:
                     comment = ""
+                # dut is a list for multi-dut testbed, convert it to string
+                if type(dut) is not str:
+                   dut = dut.__str__()
+                dut = dut.replace(",", ";")
+                dut = dut.replace(" ", "")
 
                 row = confName + "," + groupName + "," + topo + "," + ptf_image_name + "," + ptf + "," + ptf_ip + "," + ptf_ipv6 + ","+ server + "," + vm_base + "," + dut + "," + comment
                 f.write(row + "\n")
@@ -265,6 +272,8 @@ def makeSonicLabLinks(data, outfile):
             for key, item in topology.items():
                 startDevice = key
                 interfacesDetails = item.get("interfaces")
+                if not interfacesDetails:
+                    continue
 
                 for startPort, element in interfacesDetails.items():
                     startPort = startPort
@@ -377,45 +386,87 @@ def makeLab(data, devices, testbed, outfile):
                 toWrite.write("[" + key + "]\n")
                 for host in value.get("host"):
                     entry = host
+                    dev = devices.get(host.lower())
 
                     if "ptf" in key:
                         try: #get ansible host
-                            ansible_host = testbed.get(host).get("ansible").get("ansible_host")
+                            ansible_host = dev.get("ansible").get("ansible_host")
                             entry += "\tansible_host=" + ansible_host.split("/")[0]
                         except:
                             print("\t\t" + host + ": ansible_host not found")
 
                         if ansible_host:
                             try: # get ansible ssh username
-                                ansible_ssh_user = testbed.get(host.lower()).get("ansible").get("ansible_ssh_user")
+                                ansible_ssh_user = dev.get("ansible").get("ansible_ssh_user")
                                 entry += "\tansible_ssh_user=" + ansible_ssh_user
                             except:
                                 print("\t\t" + host + ": ansible_ssh_user not found")
 
                             try: # get ansible ssh pass
-                                ansible_ssh_pass = testbed.get(host.lower()).get("ansible").get("ansible_ssh_pass")
+                                ansible_ssh_pass = dev.get("ansible").get("ansible_ssh_pass")
                                 entry += "\tansible_ssh_pass=" + ansible_ssh_pass
                             except:
                                 print("\t\t" + host + ": ansible_ssh_pass not found")
                     else: #not ptf container
                         try: #get ansible host
-                            ansible_host = devices.get(host.lower()).get("ansible").get("ansible_host")
+                            ansible_host = dev.get("ansible").get("ansible_host")
                             entry += "\tansible_host=" + ansible_host.split("/")[0]
                         except:
                             print("\t\t" + host + ": ansible_host not found")
 
                         if ansible_host:
                             try: # get ansible ssh username
-                                ansible_ssh_user = devices.get(host.lower()).get("ansible").get("ansible_ssh_user")
+                                ansible_ssh_user = dev.get("ansible").get("ansible_ssh_user")
                                 entry += "\tansible_ssh_user=" + ansible_ssh_user
                             except:
                                 print("\t\t" + host + ": ansible_ssh_user not found")
 
                             try: # get ansible ssh pass
-                                ansible_ssh_pass = devices.get(host.lower()).get("ansible").get("ansible_ssh_pass")
+                                ansible_ssh_pass = dev.get("ansible").get("ansible_ssh_pass")
                                 entry += "\tansible_ssh_pass=" + ansible_ssh_pass
                             except:
                                 print("\t\t" + host + ": ansible_ssh_pass not found")
+                        try: #get hwsku
+                            hwsku = dev.get("hwsku")
+                            if hwsku is not None:
+                               entry += "\thwsku=" + hwsku
+                        except:
+                            print("\t\t" + host + ": hwsku not found")
+
+                        try: #get cardtype
+                            cardtype = dev.get("cardtype")
+                            if cardtype is not None:
+                               entry += "\tcardtype=" + cardtype
+                        except:
+                           print("\t\t" + host + ": cardtype not found")
+
+                        try: #get num_asics
+                            num_asic = dev.get("num_asic")
+                            if num_asic is not None:
+                               entry += "\tnum_asic=" + str( num_asic )
+                        except:
+                            print("\t\t" + host + " num_asic not found")
+
+                        try: #get frontend_asics
+                            frontend_asics = dev.get("frontend_asics")
+                            if frontend_asics is not None:
+                               entry += "\tfrontend_asics=" + frontend_asics.__str__()
+                        except:
+                            print("\t\t" + host + ": frontend_asics not found")
+
+                        try: #get asics_host_ip
+                            asics_host_ip = dev.get("asics_host_ip")
+                            if asics_host_ip is not None:
+                               entry += " \tasics_host_ip=" + str( asics_host_ip )
+                        except:
+                            print("\t\t" + host + " asics_host_ip not found")
+
+                        try: #get asics_host_ipv6
+                            asics_host_ipv6 = dev.get("asics_host_ipv6")
+                            if asics_host_ipv6 is not None:
+                               entry += "\tasics_host_ipv6=" + str( asics_host_ipv6 )
+                        except:
+                            print("\t\t" + host + " asics_host_ipv6 not found")
 
                     toWrite.write(entry + "\n")
                 toWrite.write("\n")
@@ -455,8 +506,13 @@ def makeVeos(data, veos, devices, outfile):
                     entry = host
 
                     try:
-                        ansible_host = devices.get(host.lower()).get("ansible").get("ansible_host")
+                        dev = devices.get(host.lower())
+                        ansible_host = dev.get("ansible").get("ansible_host")
                         entry += "\tansible_host=" + ansible_host.split("/")[0]
+                        if dev.get("device_type") == "DevSonic":
+                            entry += "\ttype=" + dev.get("type")
+                            entry += "\thwsku=" + dev.get("hwsku")
+                            entry += "\tcardtype=" + dev.get("cardtype")
                     except:
                         try:
                             ansible_host = veos.get(key).get(host).get("ansible_host")

ansible/config_sonic_basedon_testbed.yml

@@ -71,14 +71,22 @@
     delegate_to: localhost
 
   - name: find interface name mapping and individual interface speed if defined from dut
-    port_alias: hwsku="{{ hwsku }}"
+    port_alias: hwsku="{{ hwsku }}" cardtype="{{  cardtype }}"
     when: deploy is defined and deploy|bool == true
 
   - name: find interface name mapping and individual interface speed if defined with local data
-    port_alias: hwsku="{{ hwsku }}" num_asic="{{ num_asics }}"
+    port_alias: hwsku="{{ hwsku }}" num_asic="{{ num_asics }}" cardtype="{{  cardtype }}"
     delegate_to: localhost
     when: deploy is not defined or deploy|bool == false
 
+  - name: find and generate ASIC infomation
+    fabric_info:
+      hwsku: "{{ hwsku }}"
+      num_asic: "{{ num_asic }}"
+      asics_host_pfx: "{{ asics_host_ip }}"
+      asics_host_pfx6: "{{ asics_host_ipv6 }}"
+    when: deploy is defined and deploy|bool == true
+
   - name: find all enabled host_interfaces
     set_fact:
       host_if_indexes: "{{ vm_topo_config['host_interfaces_by_dut'][dut_index|int] | difference(vm_topo_config['disabled_host_interfaces_by_dut'][dut_index|int]) }}"

ansible/library/conn_graph_facts.py

@@ -163,8 +163,12 @@ def parse_graph(self):
                     deviceinfo[hostname] = {}
                     hwsku = dev.attrib['HwSku']
                     devtype = dev.attrib['Type']
+                    cardtype = "Linecard"
+                    if 'CardType' in dev.attrib:
+                        cardtype = dev.attrib['CardType']
                     deviceinfo[hostname]['HwSku'] = hwsku
                     deviceinfo[hostname]['Type'] = devtype
+                    deviceinfo[hostname]['CardType'] = cardtype
                     self.links[hostname] = {}
         devicel2info = {}
         devicel3s = self.root.find(self.dpgtag).findall('DevicesL3Info')

ansible/library/fabric_info.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+import os
+
+try:
+    from sonic_py_common import multi_asic
+except ImportError:
+    print("Failed to import multi_asic")
+
+DOCUMENTATION = '''
+module: fabric_info.py
+Ansible_version_added:  2.0.0.2
+short_description:   Find SONiC Fabric ASIC inforamtion if applicable for the DUT
+Description:
+        When the testbed has Fabric ASICs, this module helps to collect that information
+        which helps in generating the minigraph
+    Input:
+        hwsku num_asic
+
+    Return Ansible_facts:
+    fabric_info:  SONiC Fabric ASIC information
+
+'''
+
+EXAMPLES = '''
+    - name: get Fabric ASIC info
+      fabric_info: hwsku='ACS-MSN2700' num_asic=1
+'''
+
+RETURN = '''
+      ansible_facts{
+        fabric_info: [{'asicname': 'ASIC0', 'ip_prefix': '10.1.0.33/32', 'ip6_prefix': 'FC00:1::33/128'},
+                      {'asicname': 'ASIC1', 'ip_prefix': '10.1.0.34/32', 'ip6_prefix': 'FC00:1::34/128'}]
+      }
+'''
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            hwsku=dict(required=True, type='str'),
+            num_asic=dict(type='str', required=False),
+            asics_host_pfx=dict(type='str', required=False),
+            asics_host_pfx6=dict(type='str', required=False)
+        ),
+        supports_check_mode=True
+    )
+    m_args = module.params
+    try:
+        fabric_info = []
+        # num_asic may not be present for fixed systems which have no Fabric ASIC.
+        # Then return empty fabric_info
+        if 'num_asic' not in m_args or int(m_args['num_asic']) <= 1:
+           module.exit_json(ansible_facts={'fabric_info': fabric_info})
+           return
+        num_asic = int( m_args[ 'num_asic' ] )
+        asics_host_pfx = str( m_args[ 'asics_host_pfx' ] )
+        asics_host_pfx6 = str( m_args[ 'asics_host_pfx6' ] )
+        for asic_id in range(num_asic):
+            key = "ASIC%d" % asic_id
+            data = { 'asicname': key,
+                     'ip_prefix': asics_host_pfx % asic_id,
+                     'ip6_prefix': asics_host_pfx6 % asic_id }
+            fabric_info.append( data )
+        module.exit_json(ansible_facts={'fabric_info': fabric_info})
+    except (IOError, OSError), e:
+        fail_msg = "IO error" + str(e)
+        module.fail_json(msg=fail_msg)
+    except Exception, e:
+        fail_msg = "failed to find the correct fabric asic info for "+m_args['hwsku'] + str(e)
+        module.fail_json(msg=fail_msg)
+
+from ansible.module_utils.basic import *
+if __name__ == "__main__":
+    main()

ansible/library/port_alias.py

@@ -155,7 +155,8 @@ def main():
     module = AnsibleModule(
         argument_spec=dict(
             hwsku=dict(required=True, type='str'),
-            num_asic=dict(type='int', required=False)
+            num_asic=dict(type='int', required=False),
+            cardtype=dict(type='str', required=False)
         ),
         supports_check_mode=True
     )
@@ -165,6 +166,12 @@ def main():
         portmap = {}
         aliasmap = {}
         portspeed = {}
+        if 'cardtype' in m_args and m_args['cardtype'] == 'supervisor':
+           module.exit_json(ansible_facts={'port_alias': aliases,
+                                           'port_name_map': portmap,
+                                           'port_alias_map': aliasmap,
+                                           'port_speed': portspeed})
+           return
         allmap = SonicPortAliasMap(m_args['hwsku'])
         # ASIC interface names of front panel interfaces 
         front_panel_asic_ifnames = []

ansible/module_utils/port_utils.py

@@ -119,6 +119,10 @@ def get_port_alias_to_name_map(hwsku, asic_id=None):
         s100G_ports = [x for x in range(13, 21)]
 
         port_alias_to_name_map = _port_alias_to_name_map_50G(all_ports, s100G_ports)
+    elif hwsku == "Arista-7800R3-48CQ-LC" or\
+         hwsku == "Arista-7800R3K-48CQ-LC":
+         for i in range(1, 48):
+             port_alias_to_name_map["Ethernet%d/1" % i] = "Ethernet%d" % ((i - 1) * 4)
     elif hwsku == "INGRASYS-S9100-C32":
         for i in range(1, 33):
             port_alias_to_name_map["Ethernet%d/1" % i] = "Ethernet%d" % ((i - 1) * 4)

ansible/roles/vm_set/library/vm_topology.py

@@ -455,6 +455,8 @@ def bind_fp_ports(self, disconnect_vm=False):
                 br_name = OVS_FP_BRIDGE_TEMPLATE % (self.vm_names[self.vm_base_index + attr['vm_offset']], idx)
                 vm_iface = OVS_FP_TAP_TEMPLATE % (self.vm_names[self.vm_base_index + attr['vm_offset']], idx)
                 (dut_index, vlan_index, ptf_index) = VMTopology.parse_vm_vlan_port(vlan)
+                if len( self.duts_fp_ports[self.duts_name[dut_index]] ) == 0:
+                    continue
                 injected_iface = INJECTED_INTERFACES_TEMPLATE % (self.vm_set_name, ptf_index)
                 self.bind_ovs_ports(br_name, self.duts_fp_ports[self.duts_name[dut_index]][str(vlan_index)], injected_iface, vm_iface, disconnect_vm)
 

ansible/templates/minigraph_cpg.j2

@@ -1,6 +1,7 @@
   <CpgDec>
     <IsisRouters xmlns:a="http://schemas.datacontract.org/2004/07/Microsoft.Search.Autopilot.Evolution"/>
     <PeeringSessions>
+{% if cardtype is not defined or cardtype != 'supervisor' %}
 {% for index in range(vms_number) %}
 {% set vm=vms[index] %}
 {% if vm_topo_config['vm'][vm]['peer_ipv4'][dut_index|int] %}
@@ -77,9 +78,10 @@
 {% endif %}
 {% endfor %} 
 {% endfor %}
+{% endif %}
     </PeeringSessions>
     <Routers xmlns:a="http://schemas.datacontract.org/2004/07/Microsoft.Search.Autopilot.Evolution">
-{% if type is not defined or type != 'supervisor' %}
+{% if cardtype is not defined or cardtype != 'supervisor' %}
       <a:BGPRouterDeclaration>
         <a:ASN>{{ vm_topo_config['dut_asn'] }}</a:ASN>
         <a:Hostname>{{ inventory_hostname }}</a:Hostname>

ansible/templates/minigraph_device.j2

@@ -2,7 +2,7 @@
     <DeviceInfo>
       <AutoNegotiation>true</AutoNegotiation>
       <EthernetInterfaces xmlns:a="http://schemas.datacontract.org/2004/07/Microsoft.Search.Autopilot.Evolution">
-{% if type is not defined or type != 'supervisor' %}
+{% if cardtype is not defined or cardtype != 'supervisor' %}
 {% set num_of_intf = port_alias | length %}
 {% for index in range(num_of_intf) %}
         <a:EthernetInterface>

ansible/templates/minigraph_dpg.j2

@@ -2,7 +2,7 @@
     <DeviceDataPlaneInfo>
       <IPSecTunnels/>
       <LoopbackIPInterfaces xmlns:a="http://schemas.datacontract.org/2004/07/Microsoft.Search.Autopilot.Evolution">
-{% if type is not defined or type != 'supervisor' %}
+{% if cardtype is not defined or cardtype != 'supervisor' %}
         <a:LoopbackIPInterface>
           <Name>HostIP</Name>
           <AttachTo>Loopback0</AttachTo>
@@ -117,6 +117,7 @@
 {% endif %}
       </VlanInterfaces>
       <IPInterfaces>
+{% if cardtype is not defined or cardtype != 'supervisor' %}
 {% for index in range(vms_number) %}
 {% if vm_topo_config['vm'][vms[index]]['ip_intf'][dut_index|int] is not none %}
         <IPInterface>
@@ -156,11 +157,12 @@
         </IPInterface>
 {%   endif %}
 {% endfor %}
+{% endif %}
 {% endif %}
       </IPInterfaces>
       <DataAcls/>
       <AclInterfaces>
-{% if type is not defined or type != 'supervisor' %}
+{% if cardtype is not defined or cardtype != 'supervisor' %}
         <AclInterface>
           <InAcl>SNMP_ACL</InAcl>
           <AttachTo>SNMP</AttachTo>