Merge branch 'github'

            RB=1288101
            G=lnos-reviewers
            R=ntrianta,pmao,rmolina,sfardeen,zxu

* github:
  [acl_loader] Support Service ACL binding to multiple services (sonic-net#236)
  [show] Rename 'show session' to 'show mirror session' (sonic-net#235)
  [pfcstat]: create python cli tool to show pfc counters (sonic-net#233)
  [queuestat] add python CLI tool to show queue counters
  [acl-loader] Not to crash upon invalid rule (sonic-net#232)
  Show FDB type in fdbshow/show mac (sonic-net#231)
  [show] add 'show runningconfiguration all' subcommand (sonic-net#230)
  [reboot scripts] remove -t option in docker exec commands (sonic-net#228)
  [reboot] reduce stop service to only stop syncd (sonic-net#223)
  [crm]: Fix failures in CLI show commands (sonic-net#221)
  [Fast-reboot]: Gracefully shutdown syncd in fast-reboot (sonic-net#212)
  add fast-reboot support for nephos platform by stop kernel modules (sonic-net#220)
  [config bgp] Convert user input ipv6 addr to lower case before comparing (sonic-net#218)
  [PFCWD]: set default configuration when enabled by default (sonic-net#213)
  Add fast-reboot support for Aboot based images (sonic-net#214)
  sonic-utilities: Format show vlan config output (sonic-net#210)
  [AAA] Support login(ascii) authentication type (sonic-net#217)
  [sfputil] Adapt new way of getting PLATFORM(sonic-net#216)
  [Fast-Reboot]: Adapt fast-reboot-dump script for SAIv1.2  (sonic-net#211)
  Refactor fast-reboot script. Generate fast-reboot-dumps into configurable directory (sonic-net#208)
  Find correct opennsl module name before stopping it (sonic-net#207)
  [crm]: Add utility for CRM configuration (sonic-net#187)
  [reboot] update reboot script to retrieve platform with new format (sonic-net#206)
  Adapt to config engine change to load platform info properly (sonic-net#205)
  [config] Add qos clear and qos reload support (sonic-net#204)
  Dump default routes from APPL_DB table before fast-reboot (sonic-net#203)
  [acl_loader] Fix a crash issue when appdb is not consistent with cfgdb (sonic-net#202)
  [pfcwd]: add command to set pfcwd polling interval (sonic-net#192)
  [acl-loader] Prevent from hanging if run by non-root user (sonic-net#199)
  [config] Store ConfigDB init indicator boolean value as 1/0 in Redis to be language-agnostic (sonic-net#197)
  Get Vlan Id from SAI Vlan Object if bvid present (sonic-net#196)
  [TACACS+]: Fix aaa show error without configuration (sonic-net#191)
  'config bgp [shutdown|startup] neighbor <hostname>' now affects all sessions for neighbor (sonic-net#195)
  [sonic-clear] add a clear fdb command (sonic-net#186)

acl_loader/main.py

@@ -2,6 +2,7 @@
 
 import click
 import json
+import syslog
 import tabulate
 from natsort import natsorted
 
@@ -13,14 +14,17 @@
 
 def info(msg):
     click.echo(click.style("Info: ", fg='cyan') + click.style(str(msg), fg='green'))
+    syslog.syslog(syslog.LOG_INFO, msg)
 
 
 def warning(msg):
     click.echo(click.style("Warning: ", fg='cyan') + click.style(str(msg), fg='yellow'))
+    syslog.syslog(syslog.LOG_WARNING, msg)
 
 
 def error(msg):
     click.echo(click.style("Error: ", fg='cyan') + click.style(str(msg), fg='red'))
+    syslog.syslog(syslog.LOG_ERR, msg)
 
 
 def deep_update(dst, src):
@@ -80,7 +84,7 @@ def __init__(self):
         self.sessions_db_info = {}
         self.configdb = ConfigDBConnector()
         self.configdb.connect()
-        self.appdb = SonicV2Connector()
+        self.appdb = SonicV2Connector(host="127.0.0.1")
         self.appdb.connect(self.appdb.APPL_DB)
 
         self.read_tables_info()
@@ -115,8 +119,10 @@ def read_sessions_info(self):
         self.sessions_db_info = self.configdb.get_table(self.MIRROR_SESSION)
         for key in self.sessions_db_info.keys():
             app_db_info = self.appdb.get_all(self.appdb.APPL_DB, "{}:{}".format(self.MIRROR_SESSION, key))
-
-            status = app_db_info.get("status", "inactive")
+            if app_db_info:
+                status = app_db_info.get("status", "inactive")
+            else:
+                status = "error"
             self.sessions_db_info[key]["status"] = status
 
     def get_sessions_db_info(self):
@@ -352,8 +358,11 @@ def convert_rules(self):
 
             for acl_entry_name in acl_set.acl_entries.acl_entry:
                 acl_entry = acl_set.acl_entries.acl_entry[acl_entry_name]
-                rule = self.convert_rule_to_db_schema(table_name, acl_entry)
-                deep_update(self.rules_info, rule)
+                try:
+                    rule = self.convert_rule_to_db_schema(table_name, acl_entry)
+                    deep_update(self.rules_info, rule)
+                except AclLoaderException as ex:
+                    error("Error processing rule %s: %s. Skipped." % (acl_entry_name, ex))
 
             if not self.is_table_mirror(table_name):
                 deep_update(self.rules_info, self.deny_rule(table_name))
@@ -413,25 +422,34 @@ def show_table(self, table_name):
         :param table_name: Optional. ACL table name. Filter tables by specified name.
         :return:
         """
-        header = ("Name", "Type", "Ports", "Description")
+        header = ("Name", "Type", "Binding", "Description")
 
         data = []
         for key, val in self.get_tables_db_info().iteritems():
             if table_name and key != table_name:
                 continue
 
-            if not val["ports"]:
-                data.append([key, val["type"], "", val["policy_desc"]])
+            if val["type"] == AclLoader.ACL_TABLE_TYPE_CTRLPLANE:
+                services = natsorted(val["services"])
+                data.append([key, val["type"], services[0], val["policy_desc"]])
+
+                if len(services) > 1:
+                    for service in services[1:]:
+                        data.append(["", "", service, ""])
             else:
-                ports = natsorted(val["ports"])
-                data.append([key, val["type"], ports[0], val["policy_desc"]])
+                if not val["ports"]:
+                    data.append([key, val["type"], "", val["policy_desc"]])
+                else:
+                    ports = natsorted(val["ports"])
+                    data.append([key, val["type"], ports[0], val["policy_desc"]])
 
-                if len(ports) > 1:
-                    for port in ports[1:]:
-                        data.append(["", "", port, ""])
+                    if len(ports) > 1:
+                        for port in ports[1:]:
+                            data.append(["", "", port, ""])
 
         print(tabulate.tabulate(data, headers=header, tablefmt="simple", missingval=""))
 
+
     def show_session(self, session_name):
         """
         Show mirror session configuration.

clear/main.py

@@ -200,5 +200,35 @@ def arp(ipaddress):
 cli.add_command(arp)
 ip.add_command(arp)
 
+#
+# 'fdb' command ####
+#
+@cli.group()
+def fdb():
+    """Clear FDB table"""
+    pass
+
+@fdb.command('all')
+def clear_all_fdb():
+    """Clear All FDB entries"""
+    command = 'fdbclear'
+    run_command(command)
+
+# 'sonic-clear fdb port' and 'sonic-clear fdb vlan' will be added later
+'''
+@fdb.command('port')
+@click.argument('portid', required=True)
+def clear_port_fdb(portid):
+    """Clear FDB entries learned from one port"""
+    command = 'fdbclear' + ' -p ' + portid
+    run_command(command)
+
+@fdb.command('vlan')
+@click.argument('vlanid', required=True)
+def clear_vlan_fdb(vlanid):
+    """Clear FDB entries learned in one VLAN"""
+    command = 'fdbclear' + ' -v ' + vlanid
+    run_command(command)
+'''
 if __name__ == '__main__':
     cli()

config/aaa.py

@@ -124,10 +124,10 @@ def timeout(ctx, second):
 
 
 @click.command()
-@click.argument('type', metavar='<type>', type=click.Choice(["chap", "pap", "mschap"]), required=False)
+@click.argument('type', metavar='<type>', type=click.Choice(["chap", "pap", "mschap", "login"]), required=False)
 @click.pass_context
 def authtype(ctx, type):
-    """Specify TACACS+ server global auth_type [chap | pap | mschap]"""
+    """Specify TACACS+ server global auth_type [chap | pap | mschap | login]"""
     if ctx.obj == 'default':
         del_table_key('TACPLUS', 'global', 'auth_type')
     elif type:
@@ -158,7 +158,7 @@ def passkey(ctx, secret):
 @click.argument('address', metavar='<ip_address>')
 @click.option('-t', '--timeout', help='Transmission timeout interval, default 5', type=int)
 @click.option('-k', '--key', help='Shared secret')
-@click.option('-a', '--auth_type', help='Authentication type, default pap', type=click.Choice(["chap", "pap", "mschap"]))
+@click.option('-a', '--auth_type', help='Authentication type, default pap', type=click.Choice(["chap", "pap", "mschap", "login"]))
 @click.option('-o', '--port', help='TCP port range is 1 to 65535, default 49', type=click.IntRange(1, 65535), default=49)
 @click.option('-p', '--pri', help="Priority, default 1", type=click.IntRange(1, 64), default=1)
 def add(address, timeout, key, auth_type, port, pri):

config/main.py

@@ -49,38 +49,48 @@ def _get_all_neighbor_ipaddresses():
     config_db.connect()
     return config_db.get_table('BGP_NEIGHBOR').keys()
 
-def _get_neighbor_ipaddress_by_hostname(hostname):
-    """Returns string containing IP address of neighbor with hostname <hostname> or None if <hostname> not a neighbor
+def _get_neighbor_ipaddress_list_by_hostname(hostname):
+    """Returns list of strings, each containing an IP address of neighbor with
+       hostname <hostname>. Returns empty list if <hostname> not a neighbor
     """
+    addrs = []
     config_db = ConfigDBConnector()
     config_db.connect()
     bgp_sessions = config_db.get_table('BGP_NEIGHBOR')
     for addr, session in bgp_sessions.iteritems():
         if session.has_key('name') and session['name'] == hostname:
-            return addr
-    return None
+            addrs.append(addr)
+    return addrs
 
-def _switch_bgp_session_status_by_addr(ipaddress, status, verbose):
+def _change_bgp_session_status_by_addr(ipaddress, status, verbose):
     """Start up or shut down BGP session by IP address 
     """
     verb = 'Starting' if status == 'up' else 'Shutting'
     click.echo("{} {} BGP session with neighbor {}...".format(verb, status, ipaddress))
     config_db = ConfigDBConnector()
     config_db.connect()
+
     config_db.mod_entry('bgp_neighbor', ipaddress, {'admin_status': status})
 
-def _switch_bgp_session_status(ipaddr_or_hostname, status, verbose):
+def _change_bgp_session_status(ipaddr_or_hostname, status, verbose):
     """Start up or shut down BGP session by IP address or hostname
     """
-    if _is_neighbor_ipaddress(ipaddr_or_hostname):
-        ipaddress = ipaddr_or_hostname
+    ip_addrs = []
+
+    # If we were passed an IP address, convert it to lowercase because IPv6 addresses were
+    # stored in ConfigDB with all lowercase alphabet characters during minigraph parsing
+    if _is_neighbor_ipaddress(ipaddr_or_hostname.lower()):
+        ip_addrs.append(ipaddr_or_hostname.lower())
     else:
         # If <ipaddr_or_hostname> is not the IP address of a neighbor, check to see if it's a hostname
-        ipaddress = _get_neighbor_ipaddress_by_hostname(ipaddr_or_hostname)
-    if ipaddress == None:
+        ip_addrs = _get_neighbor_ipaddress_list_by_hostname(ipaddr_or_hostname)
+
+    if not ip_addrs:
         print "Error: could not locate neighbor '{}'".format(ipaddr_or_hostname)
         raise click.Abort
-    _switch_bgp_session_status_by_addr(ipaddress, status, verbose)
+
+    for ip_addr in ip_addrs:
+        _change_bgp_session_status_by_addr(ip_addr, status, verbose)
 
 def _change_hostname(hostname):
     current_hostname = os.uname()[1]
@@ -90,6 +100,41 @@ def _change_hostname(hostname):
         run_command('sed -i "/\s{}$/d" /etc/hosts'.format(current_hostname), display_cmd=True)
         run_command('echo "127.0.0.1 {}" >> /etc/hosts'.format(hostname), display_cmd=True)
 
+def _clear_qos():
+    QOS_TABLE_NAMES = [
+            'TC_TO_PRIORITY_GROUP_MAP',
+            'MAP_PFC_PRIORITY_TO_QUEUE',
+            'TC_TO_QUEUE_MAP',
+            'DSCP_TO_TC_MAP',
+            'SCHEDULER',
+            'PFC_PRIORITY_TO_PRIORITY_GROUP_MAP',
+            'PORT_QOS_MAP',
+            'WRED_PROFILE',
+            'QUEUE',
+            'CABLE_LENGTH',
+            'BUFFER_POOL',
+            'BUFFER_PROFILE',
+            'BUFFER_PG',
+            'BUFFER_QUEUE']
+    config_db = ConfigDBConnector()
+    config_db.connect()
+    for qos_table in QOS_TABLE_NAMES:
+        config_db.delete_table(qos_table)
+
+def _get_hwsku():
+    config_db = ConfigDBConnector()
+    config_db.connect()
+    metadata = config_db.get_table('DEVICE_METADATA')
+    return metadata['localhost']['hwsku']
+
+def _get_platform():
+    with open('/host/machine.conf') as machine_conf:
+        for line in machine_conf:
+            tokens = line.split('=')
+            if tokens[0].strip() == 'onie_platform' or tokens[0].strip() == 'aboot_platform':
+                return tokens[1].strip()
+    return ''
+
 # Callback for confirmation prompt. Aborts if user enters "n"
 def _abort_if_false(ctx, param, value):
     if not value:
@@ -149,7 +194,7 @@ def reload(filename, yes):
     client.flushdb()
     command = "{} -j {} --write-to-db".format(SONIC_CFGGEN_PATH, filename)
     run_command(command, display_cmd=True)
-    client.set(config_db.INIT_INDICATOR, True)
+    client.set(config_db.INIT_INDICATOR, 1)
     _restart_services()
 
 @cli.command()
@@ -186,17 +231,51 @@ def load_minigraph():
     client = config_db.redis_clients[config_db.CONFIG_DB]
     client.flushdb()
     if os.path.isfile('/etc/sonic/init_cfg.json'):
-        command = "{} -m -j /etc/sonic/init_cfg.json --write-to-db".format(SONIC_CFGGEN_PATH)
+        command = "{} -H -m -j /etc/sonic/init_cfg.json --write-to-db".format(SONIC_CFGGEN_PATH)
     else:
-        command = "{} -m --write-to-db".format(SONIC_CFGGEN_PATH)
+        command = "{} -H -m --write-to-db".format(SONIC_CFGGEN_PATH)
     run_command(command, display_cmd=True)
-    client.set(config_db.INIT_INDICATOR, True)
+    client.set(config_db.INIT_INDICATOR, 1)
+    run_command('pfcwd start_default', display_cmd=True)
     if os.path.isfile('/etc/sonic/acl.json'):
         run_command("acl-loader update full /etc/sonic/acl.json", display_cmd=True)
+    run_command("config qos reload", display_cmd=True)
     #FIXME: After config DB daemon is implemented, we'll no longer need to restart every service.
     _restart_services()
     print "Please note setting loaded from minigraph will be lost after system reboot. To preserve setting, run `config save`."
 
+#
+# 'qos' group
+#
+@cli.group()
+@click.pass_context
+def qos(ctx):
+    pass
+
+@qos.command('clear')
+def clear():
+    _clear_qos()
+
+@qos.command('reload')
+def reload():
+    _clear_qos()
+    platform = _get_platform()
+    hwsku = _get_hwsku()
+    buffer_template_file = os.path.join('/usr/share/sonic/device/', platform, hwsku, 'buffers.json.j2')
+    if os.path.isfile(buffer_template_file):
+        command = "{} -m -t {} >/tmp/buffers.json".format(SONIC_CFGGEN_PATH, buffer_template_file)
+        run_command(command, display_cmd=True)
+        command = "{} -j /tmp/buffers.json --write-to-db".format(SONIC_CFGGEN_PATH)
+        run_command(command, display_cmd=True)
+        qos_file = os.path.join('/usr/share/sonic/device/', platform, hwsku, 'qos.json')
+        if os.path.isfile(qos_file):
+            command = "{} -j {} --write-to-db".format(SONIC_CFGGEN_PATH, qos_file)
+            run_command(command, display_cmd=True)
+        else:
+            click.secho('QoS definition not found at {}'.format(qos_file), fg='yellow')
+    else:
+        click.secho('Buffer definition template not found at {}'.format(buffer_template_file), fg='yellow')
+
 #
 # 'vlan' group
 #
@@ -310,15 +389,15 @@ def all(verbose):
     """Shut down all BGP sessions"""
     bgp_neighbor_ip_list = _get_all_neighbor_ipaddresses()
     for ipaddress in bgp_neighbor_ip_list:
-        _switch_bgp_session_status_by_addr(ipaddress, 'down', verbose)
+        _change_bgp_session_status_by_addr(ipaddress, 'down', verbose)
 
 # 'neighbor' subcommand
 @shutdown.command()
 @click.argument('ipaddr_or_hostname', metavar='<ipaddr_or_hostname>', required=True)
 @click.option('-v', '--verbose', is_flag=True, help="Enable verbose output")
 def neighbor(ipaddr_or_hostname, verbose):
     """Shut down BGP session by neighbor IP address or hostname"""
-    _switch_bgp_session_status(ipaddr_or_hostname, 'down', verbose)
+    _change_bgp_session_status(ipaddr_or_hostname, 'down', verbose)
 
 @bgp.group()
 def startup():
@@ -332,15 +411,15 @@ def all(verbose):
     """Start up all BGP sessions"""
     bgp_neighbor_ip_list = _get_all_neighbor_ipaddresses()
     for ipaddress in bgp_neighbor_ip_list:
-        _switch_bgp_session_status(ipaddress, 'up', verbose)
+        _change_bgp_session_status(ipaddress, 'up', verbose)
 
 # 'neighbor' subcommand
 @startup.command()
 @click.argument('ipaddr_or_hostname', metavar='<ipaddr_or_hostname>', required=True)
 @click.option('-v', '--verbose', is_flag=True, help="Enable verbose output")
 def neighbor(ipaddr_or_hostname, verbose):
     """Start up BGP session by neighbor IP address or hostname"""
-    _switch_bgp_session_status(ipaddr_or_hostname, 'up', verbose)
+    _change_bgp_session_status(ipaddr_or_hostname, 'up', verbose)
 
 #
 # 'interface' group