Changes to support bcmsh and swss logs on multi npu platforms

Makefile.cache

@@ -68,7 +68,7 @@
 # Run the 'touch cache.skip.common' command in the base directory to exclude the common files from caching
 SONIC_COMMON_FILES_LIST :=  $(if $(wildcard cache.skip.common),, .platform slave.mk rules/functions Makefile.cache)
 SONIC_COMMON_FLAGS_LIST :=  $(CONFIGURED_PLATFORM) \
-                            $(SONIC_DPKG_CACHE_SOURCE) $(SONIC_DEBUGGING_ON) \
+                            $(SONIC_DEBUGGING_ON) \
                             $(SONIC_PROFILING_ON) $(SONIC_ENABLE_SYNCD_RPC)
 SONIC_COMMON_DPKG_LIST  :=  debian/control debian/changelog debian/rules \
                             debian/compat debian/install debian/copyright

build_debian.sh

@@ -389,61 +389,26 @@ rm /files/lib/systemd/system/rsyslog.service/Service/ExecStart/arguments
 set /files/lib/systemd/system/rsyslog.service/Service/ExecStart/arguments/1 -n
 "
 
-## Config sysctl
 sudo mkdir -p $FILESYSTEM_ROOT/var/core
+
+# Config sysctl
 sudo augtool --autosave "
 set /files/etc/sysctl.conf/kernel.core_pattern '|/usr/bin/coredump-compress %e %t %p'
-
 set /files/etc/sysctl.conf/kernel.softlockup_panic 1
 set /files/etc/sysctl.conf/kernel.panic 10
 set /files/etc/sysctl.conf/vm.panic_on_oom 2
 set /files/etc/sysctl.conf/fs.suid_dumpable 2
+" -r $FILESYSTEM_ROOT
 
-set /files/etc/sysctl.conf/net.ipv4.conf.default.forwarding 1
-set /files/etc/sysctl.conf/net.ipv4.conf.all.forwarding 1
-set /files/etc/sysctl.conf/net.ipv4.conf.eth0.forwarding 0
-
-set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_accept 0
-set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_announce 0
-set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_filter 0
-set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_notify 0
-set /files/etc/sysctl.conf/net.ipv4.conf.default.arp_ignore 0
-set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_accept 0
-set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_announce 1
-set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_filter 0
-set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_notify 1
-set /files/etc/sysctl.conf/net.ipv4.conf.all.arp_ignore 2
-
-set /files/etc/sysctl.conf/net.ipv4.neigh.default.base_reachable_time_ms 1800000
-set /files/etc/sysctl.conf/net.ipv6.neigh.default.base_reachable_time_ms 1800000
-set /files/etc/sysctl.conf/net.ipv4.neigh.default.gc_thresh1 1024
-set /files/etc/sysctl.conf/net.ipv6.neigh.default.gc_thresh1 1024
-set /files/etc/sysctl.conf/net.ipv4.neigh.default.gc_thresh2 2048
-set /files/etc/sysctl.conf/net.ipv6.neigh.default.gc_thresh2 2048
-set /files/etc/sysctl.conf/net.ipv4.neigh.default.gc_thresh3 4096
-set /files/etc/sysctl.conf/net.ipv6.neigh.default.gc_thresh3 4096
-
-set /files/etc/sysctl.conf/net.ipv6.conf.default.forwarding 1
-set /files/etc/sysctl.conf/net.ipv6.conf.all.forwarding 1
-set /files/etc/sysctl.conf/net.ipv6.conf.eth0.forwarding 0
-
-set /files/etc/sysctl.conf/net.ipv6.conf.default.accept_dad 0
-set /files/etc/sysctl.conf/net.ipv6.conf.all.accept_dad 0
-set /files/etc/sysctl.conf/net.ipv6.conf.eth0.accept_dad 0
-
-set /files/etc/sysctl.conf/net.ipv6.conf.default.keep_addr_on_down 1
-set /files/etc/sysctl.conf/net.ipv6.conf.all.keep_addr_on_down 1
-set /files/etc/sysctl.conf/net.ipv6.conf.eth0.keep_addr_on_down 1
-
-set /files/etc/sysctl.conf/net.ipv4.tcp_l3mdev_accept 1
-set /files/etc/sysctl.conf/net.ipv4.udp_l3mdev_accept 1
-
-set /files/etc/sysctl.conf/net.core.rmem_max 2097152
-set /files/etc/sysctl.conf/net.core.wmem_max 2097152
-
-set /files/etc/sysctl.conf/net.core.somaxconn 512
+sysctl_net_cmd_string=""
+while read line; do
+  [[ "$line" =~ ^#.*$ ]] && continue
+  sysctl_net_conf_key=`echo $line | awk -F '=' '{print $1}'`
+  sysctl_net_conf_value=`echo $line | awk -F '=' '{print $2}'`
+  sysctl_net_cmd_string=$sysctl_net_cmd_string"set /files/etc/sysctl.conf/$sysctl_net_conf_key $sysctl_net_conf_value"$'\n'
+done < files/image_config/sysctl/sysctl-net.conf
 
-" -r $FILESYSTEM_ROOT
+sudo augtool --autosave "$sysctl_net_cmd_string" -r $FILESYSTEM_ROOT
 
 ## docker Python API package is needed by Ansible docker module
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install 'docker==4.1.0'

build_image.sh

@@ -150,7 +150,15 @@ elif [ "$IMAGE_TYPE" = "aboot" ]; then
     if [ "$SONIC_ENABLE_IMAGE_SIGNATURE" = "y" ]; then
         TARGET_CA_CERT="$TARGET_PATH/ca.cert"
         rm -f "$TARGET_CA_CERT"
-        [ -f "$CA_CERT" ] && cp "$CA_CERT" "$TARGET_CA_CERT"
+
+        # If the ca certificate does not exist, the test certificate will be used to sign the image
+        if [ ! -f "$CA_CERT" ]; then
+            TEST_CERT_PATH=files/image_config/secureboot/test-certs
+            CA_CERT="${TEST_CERT_PATH}/ca.cert"
+            SIGNING_KEY="${TEST_CERT_PATH}/signing.key"
+            SIGNING_CERT="${TEST_CERT_PATH}/signing.cert"
+        fi
+        cp "$CA_CERT" "$TARGET_CA_CERT"
         ./scripts/sign_image.sh -i "$OUTPUT_ABOOT_IMAGE" -k "$SIGNING_KEY" -c "$SIGNING_CERT" -a "$TARGET_CA_CERT"
     fi
 else

device/arista/x86_64-arista_7050_qx32/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7050_qx32s/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7050cx3_32s/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7060_cx32s/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7060cx2_32s/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7060px4_32/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7170_32c/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7170_32cd/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7170_64c/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7260cx3_64/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_7280cr3_32p4/pmon_daemon_control.json

@@ -0,0 +1 @@
+../x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

device/arista/x86_64-arista_common/pmon_daemon_control_skip_thermalctld.json

@@ -0,0 +1,4 @@
+{
+    "skip_thermalctld": true
+}
+

dockers/docker-database/Dockerfile.j2

@@ -40,6 +40,7 @@ COPY ["docker-database-init.sh", "/usr/local/bin/"]
 COPY ["database_config.json.j2", "/usr/share/sonic/templates/"]
 COPY ["database_global.json.j2", "/usr/share/sonic/templates/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["files/sysctl-net.conf", "/etc/sysctl.d/"]
 COPY ["critical_processes", "/etc/supervisor"]
 
 ENTRYPOINT ["/usr/local/bin/docker-database-init.sh"]

dockers/docker-database/database_config.json.j2

@@ -1,7 +1,7 @@
 {
     "INSTANCES": {
         "redis":{
-            "hostname" : "127.0.0.1",
+            "hostname" : "{{HOST_IP}}",
             "port" : 6379,
             "unix_socket_path" : "/var/run/redis{{NAMESPACE_ID}}/redis.sock",
             "persistence_for_warm_boot" : "yes"

dockers/docker-database/docker-database-init.sh

@@ -1,12 +1,29 @@
 #!/usr/bin/env bash
 
+# For linux host namespace, in both single and multi ASIC platform use the loopback interface
+# For other namespaces, use eth0 interface which is connected to the docker0 bridge in the host.
+if [[ $NAMESPACE_ID == "" ]]
+then
+    INTFC=lo
+else
+    INTFC=eth0
+fi
+
+# Get the ip address of the interface
+# if the ip address was not retrieved correctly, put localhost(127.0.0.1) as the default.
+host_ip=$(ip -4 -o addr show $INTFC | awk '{print $4}' | cut -d'/' -f1 | head -1)
+if [[ $host_ip == "" ]]
+then
+    host_ip=127.0.0.1
+fi
+
 REDIS_DIR=/var/run/redis$NAMESPACE_ID
 mkdir -p $REDIS_DIR/sonic-db
 
 if [ -f /etc/sonic/database_config$NAMESPACE_ID.json ]; then
     cp /etc/sonic/database_config$NAMESPACE_ID.json $REDIS_DIR/sonic-db/database_config.json
 else
-    j2 /usr/share/sonic/templates/database_config.json.j2 > $REDIS_DIR/sonic-db/database_config.json
+    HOST_IP=$host_ip j2 /usr/share/sonic/templates/database_config.json.j2 > $REDIS_DIR/sonic-db/database_config.json
 fi
 
 mkdir -p /etc/supervisor/conf.d/

dockers/docker-database/supervisord.conf.j2

@@ -20,7 +20,12 @@ stderr_logfile=syslog
 {% if INSTANCES %}
 {%     for redis_inst, redis_items in INSTANCES.iteritems() %}
 [program: {{ redis_inst }}]
-command=/bin/bash -c "{ [[ -s /var/lib/{{ redis_inst }}/dump.rdb ]] || rm -f /var/lib/{{ redis_inst }}/dump.rdb; } && mkdir -p /var/lib/{{ redis_inst }} && exec /usr/bin/redis-server /etc/redis/redis.conf --port {{ redis_items['port'] }} --unixsocket {{ redis_items['unix_socket_path'] }} --pidfile /var/run/redis/{{ redis_inst }}.pid --dir /var/lib/{{ redis_inst }}"
+{% if redis_items['hostname'] != '127.0.0.1' %}
+{%- set LOOPBACK_IP = '127.0.0.1' -%}
+{%- else -%}
+{%- set LOOPBACK_IP = '' -%}
+{%- endif -%}
+command=/bin/bash -c "{ [[ -s /var/lib/{{ redis_inst }}/dump.rdb ]] || rm -f /var/lib/{{ redis_inst }}/dump.rdb; } && mkdir -p /var/lib/{{ redis_inst }} && exec /usr/bin/redis-server /etc/redis/redis.conf --bind {{ LOOPBACK_IP }} {{ redis_items['hostname'] }} --port {{ redis_items['port'] }} --unixsocket {{ redis_items['unix_socket_path'] }} --pidfile /var/run/redis/{{ redis_inst }}.pid --dir /var/lib/{{ redis_inst }}"
 priority=2
 autostart=true
 autorestart=false

dockers/docker-fpm-frr/frr/bgpd/bgpd.main.conf.j2

@@ -1,3 +1,4 @@
+{% from "common/functions.conf.j2" import get_ipv4_loopback_address, get_ipv6_loopback_address %}
 !
 ! template: bgpd/bgpd.main.conf.j2
 !

dockers/docker-orchagent/Dockerfile.j2

@@ -1,5 +1,5 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-config-engine-stretch
+FROM docker-config-engine-buster
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf

files/build_templates/docker_image_ctl.j2

@@ -27,6 +27,22 @@ link_namespace() {
 }
 {%- endif %}
 
+function updateSyslogConf()
+{
+    # On multiNPU platforms, change the syslog target ip to docker0 ip to allow logs from containers
+    # running on the namespace to reach the rsyslog service running on the host
+    # Also update the container name  
+    if [[ ($NUM_ASIC -gt 1) ]]; then
+        TARGET_IP=$(docker network inspect bridge --format={{ "'{{(index .IPAM.Config 0).Gateway}}'" }})
+        CONTAINER_NAME="{{docker_container_name}}$DEV"
+        TMP_FILE="/tmp/rsyslog.$CONTAINER_NAME.conf"
+
+        sonic-cfggen -t /usr/share/sonic/templates/rsyslog-container.conf.j2 -a "{\"target_ip\": \"$TARGET_IP\", \"container_name\": \"$CONTAINER_NAME\" }"  > $TMP_FILE
+        docker cp $TMP_FILE {{docker_container_name}}$DEV:/etc/rsyslog.conf
+        rm -rf $TMP_FILE
+    fi
+}
+
 function getMountPoint()
 {
     echo $1 | python -c "import sys, json, os; mnts = [x for x in json.load(sys.stdin)[0]['Mounts'] if x['Destination'] == '/usr/share/sonic/hwsku']; print '' if len(mnts) == 0 else os.path.basename(mnts[0]['Source'])" 2>/dev/null
@@ -68,13 +84,14 @@ function preStartAction()
 {%- else %}
     : # nothing
 {%- endif %}
+    updateSyslogConf
 }
 
 function postStartAction()
 {
 {%- if docker_container_name == "database" %}
     if [ "$DEV" ]; then
-        docker exec -i database$DEV sysctl -w net.ipv6.conf.all.disable_ipv6=0
+        docker exec -i database$DEV sysctl --system -e
         link_namespace $DEV
     fi
 
@@ -282,6 +299,14 @@ start() {
         --tmpfs /tmp \
 {%- endif %}
 {%- endif %}
+{%- if sonic_asic_platform == "broadcom" %}
+{%- if docker_container_name == "syncd" %}
+        -v /var/run/docker-syncd$DEV:/var/run/sswsyncd \
+{%- endif %}
+{%- endif %}
+{%- if docker_container_name == "swss" %}
+       -v /var/log/swss$DEV:/var/log/swss:rw \
+{%- endif %}
 {%- if docker_container_name == "bgp" %}
         -v /etc/sonic/frr/$DEV:/etc/frr:rw \
 {%- endif %}

files/build_templates/sonic_debian_extension.j2

@@ -241,6 +241,7 @@ echo "warmboot-finalizer.service" | sudo tee -a $GENERATED_SERVICE_FILE
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog-config.service $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog-config.sh $FILESYSTEM_ROOT/usr/bin/
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.conf.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
+sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog-container.conf.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.d/* $FILESYSTEM_ROOT/etc/rsyslog.d/
 echo "rsyslog-config.service" | sudo tee -a $GENERATED_SERVICE_FILE
 
@@ -251,7 +252,7 @@ sudo cp $IMAGE_CONFIGS/syslog/host_umount.sh $FILESYSTEM_ROOT/usr/bin/
 
 # Copy logrotate.d configuration files
 sudo cp -f $IMAGE_CONFIGS/logrotate/logrotate.d/* $FILESYSTEM_ROOT/etc/logrotate.d/
-
+sudo cp $IMAGE_CONFIGS/logrotate/logrotate-rsyslog.j2 $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/
 # Copy systemd-journald configuration files
 sudo cp -f $IMAGE_CONFIGS/systemd/journald.conf $FILESYSTEM_ROOT/etc/systemd/
 

files/image_config/logrotate/logrotate.d/rsyslog → files/image_config/logrotate/logrotate-rsyslog.j2

@@ -30,8 +30,15 @@
 /var/log/telemetry.log
 /var/log/quagga/bgpd.log
 /var/log/quagga/zebra.log
+{% if namespaces > 1 %}
+{% for ns in range(namespaces) %}
+/var/log/swss{{ns}}/sairedis.rec
+/var/log/swss{{ns}}/swss.rec
+{% endfor %}
+{% else %}
 /var/log/swss/sairedis.rec
 /var/log/swss/swss.rec
+{% endif %}
 {
     size 1M
     rotate 5000
@@ -84,10 +91,20 @@
         done
     endscript
     postrotate
+{% if namespaces > 1 %}
+{% for ns in range(namespaces) %} 
+        if [ $(echo $1 | grep -c "/var/log/swss{{ns}}/") -gt 0 ]; then
+            pgrep -x orchagent | xargs /bin/kill -HUP 2>/dev/null || true
+        else
+            /bin/kill -HUP $(cat /var/run/rsyslogd.pid)
+        fi
+{% endfor %}
+{% else %}
         if [ $(echo $1 | grep -c "/var/log/swss/") -gt 0 ]; then
             pgrep -x orchagent | xargs /bin/kill -HUP 2>/dev/null || true
         else
             /bin/kill -HUP $(cat /var/run/rsyslogd.pid)
         fi
+{% endif %}
     endscript
 }

files/image_config/rsyslog/rsyslog-config.sh

@@ -1,4 +1,26 @@
 #!/bin/bash
 
-sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 >/etc/rsyslog.conf
+PLATFORM=`sonic-cfggen -H -v DEVICE_METADATA.localhost.platform`
+
+# Parse the device specific asic conf file, if it exists
+ASIC_CONF=/usr/share/sonic/device/$PLATFORM/asic.conf
+if [ -f "$ASIC_CONF" ]; then
+    source $ASIC_CONF
+fi
+
+# On Multi NPU platforms we need to start  the rsyslog server on the docker0 ip address
+# for the syslogs from the containers in the namespaces to work.
+# on Single NPU platforms we continue to use loopback adddres
+
+if [[ ($NUM_ASIC -gt 1) ]]; then
+    udp_server_ip=$(ip -o -4 addr list docker0 | awk '{print $4}' | cut -d/ -f1)
+else
+    udp_server_ip=$(ip -o -4 addr list lo scope host | awk '{print $4}' | cut -d/ -f1)
+fi
+
+sonic-cfggen -d -t /usr/share/sonic/templates/rsyslog.conf.j2 -a "{\"udp_server_ip\": \"$udp_server_ip\"}"  >/etc/rsyslog.conf
+
+#Render the logrotate config files for rsyslog as well.
+sonic-cfggen -d -t /usr/share/sonic/templates/logrotate-rsyslog.j2 -a "{\"namespaces\": $NUM_ASIC}" > /etc/logrotate.d/rsyslog
+
 systemctl restart rsyslog