[build]: combine feature and container feature table (#5081)

1. remove container feature table
2. do not generate feature entry if the feature is not included
   in the image
3. rename ENABLE_* to INCLUDE_* for better clarity
4. rename feature status to feature state
5. [submodule]: update sonic-utilities

* 9700e45 2020-08-03 | [show/config]: combine feature and container feature cli (#1015) (HEAD, origin/master, origin/HEAD) [lguohan]
* c9d3550 2020-08-03 | [tests]: fix drops_group_test failure on second run (#1023) [lguohan]
* dfaae69 2020-08-03 | [lldpshow]: Fix input device is not a TTY error (#1016) [Arun Saravanan Balachandran]
* 216688e 2020-08-02 | [tests]: rename sonic-utilitie-tests to tests (#1022) [lguohan]

Signed-off-by: Guohan Lu <lguohan@gmail.com>

Makefile.work

@@ -9,7 +9,7 @@
 #    through http.
 #  * ENABLE_ZTP: Enables zero touch provisioning.
 #  * SHUTDOWN_BGP_ON_START: Sets admin-down state for all bgp peerings after restart.
-#  * INSTALL_KUBERNETES: Allows including Kubernetes
+#  * INCLUDE_KUBERNETES: Allows including Kubernetes
 #  * ENABLE_PFCWD_ON_START: Enable PFC Watchdog (PFCWD) on server-facing ports
 #  * by default for TOR switch.
 #  * ENABLE_SYNCD_RPC: Enables rpc-based syncd builds.
@@ -188,7 +188,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            ENABLE_DHCP_GRAPH_SERVICE=$(ENABLE_DHCP_GRAPH_SERVICE) \
                            ENABLE_ZTP=$(ENABLE_ZTP) \
                            SHUTDOWN_BGP_ON_START=$(SHUTDOWN_BGP_ON_START) \
-                           INSTALL_KUBERNETES=$(INSTALL_KUBERNETES) \
+                           INCLUDE_KUBERNETES=$(INCLUDE_KUBERNETES) \
                            KUBERNETES_VERSION=$(KUBERNETES_VERSION) \
                            KUBERNETES_CNI_VERSION=$(KUBERNETES_CNI_VERSION) \
                            K8s_GCR_IO_PAUSE_VERSION=$(K8s_GCR_IO_PAUSE_VERSION) \
@@ -207,8 +207,8 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_DPKG_CACHE_SOURCE=$(SONIC_DPKG_CACHE_SOURCE) \
                            HTTP_PROXY=$(http_proxy) \
                            HTTPS_PROXY=$(https_proxy) \
-                           SONIC_ENABLE_SYSTEM_TELEMETRY=$(ENABLE_SYSTEM_TELEMETRY) \
-                           SONIC_ENABLE_RESTAPI=$(ENABLE_RESTAPI) \
+                           SONIC_INCLUDE_SYSTEM_TELEMETRY=$(INCLUDE_SYSTEM_TELEMETRY) \
+                           SONIC_INCLUDE_RESTAPI=$(INCLUDE_RESTAPI) \
                            TELEMETRY_WRITABLE=$(TELEMETRY_WRITABLE) \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \

build_debian.sh

@@ -212,7 +212,7 @@ fi
 
 sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y remove software-properties-common gnupg2
 
-if [ "$INSTALL_KUBERNETES" == "y" ]
+if [ "$INCLUDE_KUBERNETES" == "y" ]
 then
     ## Install Kubernetes
     echo '[INFO] Install kubernetes'

files/build_scripts/mask_disabled_services.py

@@ -9,5 +9,5 @@
     init_cfg = json.load(init_cfg_file)
     if 'FEATURE' in init_cfg:
         for feature_name, feature_props in init_cfg['FEATURE'].items():
-            if 'status' in feature_props and feature_props['status'] == 'disabled':
+            if 'state' in feature_props and feature_props['state'] == 'disabled':
                 subprocess.run(['systemctl', 'mask', '{}.service'.format(feature_name)])

files/build_templates/init_cfg.json.j2

@@ -17,18 +17,27 @@
 {% endfor %}
         }
     },
+{%- set features = [("bgp", "enabled", "enabled"),
+                   ("database", "enabled", "disabled"),
+                   ("dhcp_relay", "enabled", "enabled"),
+                   ("lldp", "enabled", "enabled"),
+                   ("pmon", "enabled", "enabled"),
+                   ("radv", "enabled", "enabled"),
+                   ("snmp", "enabled", "enabled"),
+                   ("swss", "enabled", "enabled"),
+                   ("syncd", "enabled", "enabled"),
+                   ("teamd", "enabled", "enabled")] %}
+{%- if include_iccpd == "y" %}{% do features.append(("iccpd", "disabled", "enabled")) %}{% endif %}
+{%- if include_mgmt_framework == "y" %}{% do features.append(("mgmt-framework", "enabled", "enabled")) %}{% endif %}
+{%- if include_nat == "y" %}{% do features.append(("nat", "disabled", "enabled")) %}{% endif %}
+{%- if include_restapi == "y" %}{% do features.append(("restapi", "disabled", "enabled")) %}{% endif %}
+{%- if include_sflow == "y" %}{% do features.append(("sflow", "disabled", "enabled")) %}{% endif %}
+{%- if include_system_telemetry == "y" %}{% do features.append(("telemetry", "enabled", "enabled")) %}{% endif %}
     "FEATURE": {
-{%- for feature, status in [("sflow", "disabled"), ("telemetry", "enabled")] %}
+{%- for feature, state, autorestart in features %}
         "{{feature}}": {
-            "status": "{{status}}"
-        }{% if not loop.last %},{% endif -%}
-{% endfor %}
-    },
-    "CONTAINER_FEATURE": {
-{%- for container in ["bgp", "database", "dhcp_relay", "lldp", "nat", "pmon", "radv", "restapi", "sflow",
-                      "snmp", "swss", "syncd", "teamd", "telemetry"] %}
-        "{{container}}": {
-            "auto_restart": "{% if container == "database" %}disabled{% else %}enabled{% endif %}",
+            "state": "{{state}}",
+            "auto_restart": "{{autorestart}}",
             "high_mem_alert": "disabled"
         }{% if not loop.last %},{% endif -%}
 {% endfor %}

files/build_templates/sonic_debian_extension.j2

@@ -307,7 +307,7 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install azure-s
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install watchdog==0.10.2
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install futures==3.3.0
 
-{% if install_kubernetes == "y" %}
+{% if include_kubernetes == "y" %}
 # Copy kubelet service files
 # Keep it disabled until join, else it continuously restart and as well spew too many
 # non-required log lines wasting syslog resources.
@@ -481,7 +481,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT docker $SONIC_NATIVE_DOCKERD_FOR_DOCKERFS ta
 {% endif %}
 {% endfor %}
 
-{% if install_kubernetes == "y" %}
+{% if include_kubernetes == "y" %}
 ## Pull in kubernetes docker images
 echo "pulling universal k8s images ..."
 FLANNEL_ARCH=$([ "${CONFIGURED_ARCH}" == "armhf" ] && echo "arm64" || echo "${CONFIGURED_ARCH}")
@@ -537,7 +537,7 @@ sudo LANG=C cp $SCRIPTS_DIR/sonic-netns-exec $FILESYSTEM_ROOT/usr/bin/sonic-netn
 # It implements delayed start of services
 sudo cp $BUILD_TEMPLATES/snmp.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
 echo "snmp.timer" | sudo tee -a $GENERATED_SERVICE_FILE
-{% if enable_system_telemetry == 'y' %}
+{% if include_system_telemetry == 'y' %}
 sudo cp $BUILD_TEMPLATES/telemetry.timer $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
 echo "telemetry.timer" | sudo tee -a $GENERATED_SERVICE_FILE
 {% endif %}

files/image_config/hostcfgd/hostcfgd

@@ -41,8 +41,8 @@ def obfuscate(data):
         return data
 
 
-def update_feature_status(feature_name, status):
-    if status == "enabled":
+def update_feature_state(feature_name, state):
+    if state == "enabled":
         start_cmds = []
         start_cmds.append("sudo systemctl unmask {}.service".format(feature_name))
         start_cmds.append("sudo systemctl enable {}.service".format(feature_name))
@@ -56,7 +56,7 @@ def update_feature_status(feature_name, status):
                               .format(err.cmd, err.returncode, err.output))
                 continue
         syslog.syslog(syslog.LOG_INFO, "Feature '{}' is enabled and started".format(feature_name))
-    elif status == "disabled":
+    elif state == "disabled":
         stop_cmds = []
         stop_cmds.append("sudo systemctl stop {}.service".format(feature_name))
         stop_cmds.append("sudo systemctl disable {}.service".format(feature_name))
@@ -71,7 +71,7 @@ def update_feature_status(feature_name, status):
                 continue
         syslog.syslog(syslog.LOG_INFO, "Feature '{}' is stopped and disabled".format(feature_name))
     else:
-        syslog.syslog(syslog.LOG_ERR, "Unexpected status value '{}' for feature '{}'".format(status, feature_name))
+        syslog.syslog(syslog.LOG_ERR, "Unexpected state value '{}' for feature '{}'".format(state, feature_name))
 
 
 class Iptables(object):
@@ -272,19 +272,19 @@ class HostConfigDaemon:
         self.iptables = Iptables()
         self.iptables.load(lpbk_table)
 
-    def update_all_feature_statuses(self):
+    def update_all_feature_states(self):
         feature_table = self.config_db.get_table('FEATURE')
         for feature_name in feature_table.keys():
             if not feature_name:
                 syslog.syslog(syslog.LOG_WARNING, "Feature is None")
                 continue
 
-            status = feature_table[feature_name]['status']
-            if not status:
-                syslog.syslog(syslog.LOG_WARNING, "Status of feature '{}' is None".format(feature_name))
+            state = feature_table[feature_name]['state']
+            if not state:
+                syslog.syslog(syslog.LOG_WARNING, "Eanble state of feature '{}' is None".format(feature_name))
                 continue
 
-            update_feature_status(feature_name, status)
+            update_feature_state(feature_name, state)
 
     def aaa_handler(self, key, data):
         self.aaacfg.aaa_update(key, data)
@@ -314,29 +314,29 @@ class HostConfigDaemon:
 
         self.iptables.iptables_handler(key, data, add)
 
-    def feature_status_handler(self, key, data):
+    def feature_state_handler(self, key, data):
         feature_name = key
         feature_table = self.config_db.get_table('FEATURE')
         if feature_name not in feature_table.keys():
             syslog.syslog(syslog.LOG_WARNING, "Feature '{}' not in FEATURE table".format(feature_name))
             return
 
-        status = feature_table[feature_name]['status']
-        if not status:
-            syslog.syslog(syslog.LOG_WARNING, "Status of feature '{}' is None".format(feature_name))
+        state = feature_table[feature_name]['state']
+        if not state:
+            syslog.syslog(syslog.LOG_WARNING, "Enable state of feature '{}' is None".format(feature_name))
             return
 
-        update_feature_status(feature_name, status)
+        update_feature_state(feature_name, state)
 
     def start(self):
-        # Update all feature statuses once upon starting
-        self.update_all_feature_statuses()
+        # Update all feature states once upon starting
+        self.update_all_feature_states()
 
         self.config_db.subscribe('AAA', lambda table, key, data: self.aaa_handler(key, data))
         self.config_db.subscribe('TACPLUS_SERVER', lambda table, key, data: self.tacacs_server_handler(key, data))
         self.config_db.subscribe('TACPLUS', lambda table, key, data: self.tacacs_global_handler(key, data))
         self.config_db.subscribe('LOOPBACK_INTERFACE', lambda table, key, data: self.lpbk_handler(key, data))
-        self.config_db.subscribe('FEATURE', lambda table, key, data: self.feature_status_handler(key, data))
+        self.config_db.subscribe('FEATURE', lambda table, key, data: self.feature_state_handler(key, data))
         self.config_db.listen()
 
 

files/scripts/supervisor-proc-exit-listener

@@ -11,21 +11,20 @@ import swsssdk
 from supervisor import childutils
 
 # Each line of this file should specify either one critical process or one
-# critical process group, (as defined in supervisord.conf file), in the 
+# critical process group, (as defined in supervisord.conf file), in the
 # following format:
 #
 # program:<process_name>
 # group:<group_name>
 CRITICAL_PROCESSES_FILE = '/etc/supervisor/critical_processes'
 
-# This table in databse contains the features for container and each
-# feature for a row will be configured a state or number.
-CONTAINER_FEATURE_TABLE_NAME = 'CONTAINER_FEATURE'
+# The FEATURE table in config db contains auto-restart field
+FEATURE_TABLE_NAME = 'FEATURE'
 
 # Read the critical processes/group names from CRITICAL_PROCESSES_FILE
 def get_critical_group_and_process_list():
     critical_group_list = []
-    critical_process_list = [] 
+    critical_process_list = []
 
     with open(CRITICAL_PROCESSES_FILE, 'r') as file:
 	for line in file:
@@ -43,7 +42,7 @@ def get_critical_group_and_process_list():
             else:
               syslog.syslog(syslog.LOG_ERR, "Syntax of the line {} in critical_processes file is incorrect. Exiting...".format(line))
               sys.exit(6)
-     
+
     return critical_group_list, critical_process_list
 
 def main(argv):
@@ -82,23 +81,23 @@ def main(argv):
             if container_name != 'database':
                 config_db = swsssdk.ConfigDBConnector()
                 config_db.connect()
-                container_features_table = config_db.get_table(CONTAINER_FEATURE_TABLE_NAME)
-                if not container_features_table:
-                    syslog.syslog(syslog.LOG_ERR, "Unable to retrieve container features table from Config DB. Exiting...")
+                features_table = config_db.get_table(FEATURE_TABLE_NAME)
+                if not features_table:
+                    syslog.syslog(syslog.LOG_ERR, "Unable to retrieve features table from Config DB. Exiting...")
                     sys.exit(2)
 
-                if not container_features_table.has_key(container_name):
-                    syslog.syslog(syslog.LOG_ERR, "Unable to retrieve features for container '{}'. Exiting...".format(container_name))
+                if not features_table.has_key(container_name):
+                    syslog.syslog(syslog.LOG_ERR, "Unable to retrieve feature '{}'. Exiting...".format(container_name))
                     sys.exit(3)
 
-                restart_feature = container_features_table[container_name].get('auto_restart')
+                restart_feature = features_table[container_name].get('auto_restart')
                 if not restart_feature:
-                    syslog.syslog(syslog.LOG_ERR, "Unable to determine auto-restart feature status for container '{}'. Exiting...".format(container_name))
+                    syslog.syslog(syslog.LOG_ERR, "Unable to determine auto-restart feature status for '{}'. Exiting...".format(container_name))
                     sys.exit(4)
 
-            # If container is database or auto-restart feature is enabled and at the same time 
+            # If container is database or auto-restart feature is enabled and at the same time
             # a critical process exited unexpectedly, terminate supervisor
-            if ((container_name == 'database' or restart_feature == 'enabled') and expected == 0 and 
+            if ((container_name == 'database' or restart_feature == 'enabled') and expected == 0 and
                     (processname in critical_process_list or groupname in critical_group_list)):
                 MSG_FORMAT_STR = "Process {} exited unxepectedly. Terminating supervisor..."
                 msg = MSG_FORMAT_STR.format(payload_headers['processname'])

platform/marvell-arm64/rules.mk

@@ -7,7 +7,7 @@ include $(PLATFORM_PATH)/libsaithrift-dev.mk
 include $(PLATFORM_PATH)/docker-ptf-mrvl.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/linux-kernel-arm64.mk
-ENABLE_SYSTEM_TELEMETRY = ""
+INCLUDE_SYSTEM_TELEMETRY = ""
 
 
 SONIC_ALL += $(SONIC_ONE_IMAGE) \

platform/marvell-armhf/rules.mk

@@ -9,7 +9,7 @@ include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/linux-kernel-armhf.mk
 include $(PLATFORM_PATH)/platform-et6448m.mk
 
-ENABLE_SYSTEM_TELEMETRY = ""
+INCLUDE_SYSTEM_TELEMETRY = ""
 ENABLE_SYNCD_RPC = ""
 
 SONIC_ALL += $(SONIC_ONE_IMAGE) \

rules/config

@@ -83,9 +83,6 @@ ENABLE_ORGANIZATION_EXTENSIONS = y
 #SONIC_DEBUGGING_ON = y
 #SONIC_PROFILING_ON = y
 
-# ENABLE_SYSTEM_TELEMETRY - build docker-sonic-telemetry for system telemetry support
-ENABLE_SYSTEM_TELEMETRY = y
-
 # DEFAULT_KERNEL_PROCURE_METHOD - default method for obtaining kernel
 #   build:    build kernel from source
 #   download: download pre-built kernel from Azure storage.
@@ -116,32 +113,35 @@ SONIC_DPKG_CACHE_SOURCE ?= /var/cache/sonic/artifacts
 DEFAULT_VS_PREPARE_MEM = yes
 
 
-# ENABLE_ICCPD - build docker-iccpd for mclag support
-ENABLE_ICCPD = n
+# INCLUDE_SYSTEM_TELEMETRY - build docker-sonic-telemetry for system telemetry support
+INCLUDE_SYSTEM_TELEMETRY = y
+
+# INCLUDE_ICCPD - build docker-iccpd for mclag support
+INCLUDE_ICCPD = n
 
-# ENABLE_SYSTEM_SFLOW - build docker-sonic-sflow for sFlow support
-ENABLE_SFLOW = y
+# INCLUDE_SFLOW - build docker-sflow for sFlow support
+INCLUDE_SFLOW = y
 
-# ENABLE_MGMT_FRAMEWORK - build docker-sonic-mgt-framework for CLI and REST server support
-ENABLE_MGMT_FRAMEWORK = y
+# INCLUDE_MGMT_FRAMEWORK - build docker-sonic-mgmt-framework for CLI and REST server support
+INCLUDE_MGMT_FRAMEWORK = y
 
-# ENABLE_RESTAPI - build docker-sonic-restapi for configuring the switch using REST APIs
-ENABLE_RESTAPI = n
+# INCLUDE_RESTAPI - build docker-sonic-restapi for configuring the switch using REST APIs
+INCLUDE_RESTAPI = n
 
-# ENABLE_NAT - build docker-sonic-nat for nat support
-ENABLE_NAT = y
+# INCLUDE_NAT - build docker-nat for nat support
+INCLUDE_NAT = y
 
 # TELEMETRY_WRITABLE - Enable write/config operations via the gNMI interface.
 # Uncomment to enable:
 # TELEMETRY_WRITABLE = y
-# INSTALL_KUBERNETES - if set to y kubernetes packages are installed to be able to
+# INCLUDE_KUBERNETES - if set to y kubernetes packages are installed to be able to
 # run as worker node in kubernetes cluster.
-INSTALL_KUBERNETES = n
+INCLUDE_KUBERNETES = n
 
 # KUBERNETES_VERSION - Set to the required version.
 # K8s_GCR_IO_PAUSE_VERSION - Version of k8s universal pause container image
 # K8s_CNI_FLANNEL_VERSION - Flannel used as CNI; Appropriate version for this Kubernetes version
-# These are Used *only* when INSTALL_KUBERNETES=y
+# These are Used *only* when INCLUDE_KUBERNETES=y
 # NOTE: As a worker node it has to run version compatible to kubernetes master.
 #
 KUBERNETES_VERSION = 1.18.6

rules/docker-iccpd.mk

@@ -10,7 +10,7 @@ $(DOCKER_ICCPD)_DBG_DEPENDS += $(SWSS_DBG) $(LIBSWSSCOMMON_DBG) $(ICCPD_DBG)
 $(DOCKER_ICCPD)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES)
 $(DOCKER_ICCPD)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BUSTER)
 
-ifeq ($(ENABLE_ICCPD), y)
+ifeq ($(INCLUDE_ICCPD), y)
 SONIC_DOCKER_IMAGES += $(DOCKER_ICCPD)
 SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_ICCPD)
 SONIC_DOCKER_DBG_IMAGES += $(DOCKER_ICCPD_DBG)

rules/docker-nat.mk

@@ -13,12 +13,12 @@ $(DOCKER_NAT)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PA
 
 $(DOCKER_NAT)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BUSTER)
 
-ifeq ($(ENABLE_NAT), y)
+ifeq ($(INCLUDE_NAT), y)
 SONIC_DOCKER_IMAGES += $(DOCKER_NAT)
 SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_NAT)
 endif
 
-ifeq ($(ENABLE_NAT), y)
+ifeq ($(INCLUDE_NAT), y)
 SONIC_DOCKER_DBG_IMAGES += $(DOCKER_NAT_DBG)
 SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_NAT_DBG)
 endif

rules/docker-restapi.mk

@@ -10,7 +10,7 @@ $(DOCKER_RESTAPI)_PATH = $(DOCKERS_PATH)/$(DOCKER_RESTAPI_STEM)
 
 $(DOCKER_RESTAPI)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH)
 
-ifeq ($(ENABLE_RESTAPI), y)
+ifeq ($(INCLUDE_RESTAPI), y)
 SONIC_DOCKER_IMAGES += $(DOCKER_RESTAPI)
 SONIC_STRETCH_DOCKERS += $(DOCKER_RESTAPI)
 SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_RESTAPI)

rules/docker-sflow.mk

@@ -14,12 +14,12 @@ $(DOCKER_SFLOW)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_
 $(DOCKER_SFLOW)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BUSTER)
 
 SONIC_DOCKER_IMAGES += $(DOCKER_SFLOW)
-ifeq ($(ENABLE_SFLOW), y)
+ifeq ($(INCLUDE_SFLOW), y)
 SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_SFLOW)
 endif
 
 SONIC_DOCKER_DBG_IMAGES += $(DOCKER_SFLOW_DBG)
-ifeq ($(ENABLE_SFLOW), y)
+ifeq ($(INCLUDE_SFLOW), y)
 SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_SFLOW_DBG)
 endif