[System logs]: Improvements to prevent filling /var/log partition

.gitignore

@@ -1,5 +1,6 @@
 # Build system related
 .platform
+.screen
 
 # Build artifacts
 fsroot/

files/build_templates/sonic_debian_extension.j2

@@ -106,7 +106,7 @@ sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.conf.j2 $FILESYSTEM_ROOT/usr/share/sonic/
 sudo cp $IMAGE_CONFIGS/rsyslog/rsyslog.d/* $FILESYSTEM_ROOT/etc/rsyslog.d/
 
 # Copy logrotate.d configuration files
-sudo cp -f $IMAGE_CONFIGS/logrotate.d/rsyslog $FILESYSTEM_ROOT/etc/logrotate.d/
+sudo cp -f $IMAGE_CONFIGS/logrotate/logrotate.d/* $FILESYSTEM_ROOT/etc/logrotate.d/
 
 # Copy systemd-journald configuration files
 sudo cp -f $IMAGE_CONFIGS/systemd/journald.conf $FILESYSTEM_ROOT/etc/systemd/

files/image_config/cron.d/logrotate

@@ -1,2 +1,2 @@
 # Attempt to rotate system logs once per minute
-* * * * * root /usr/sbin/logrotate /etc/logrotate.d/rsyslog
+* * * * * root /usr/sbin/logrotate /etc/logrotate.conf > /dev/null 2>&1

files/image_config/logrotate/logrotate.conf

@@ -0,0 +1,32 @@
+# see "man logrotate" for details
+# Rotate log files daily by default
+daily
+
+# Keep 4 days worth of backlogs by default
+rotate 4
+
+# create new (empty) log files after rotating old ones
+create
+
+# uncomment this if you want your log files compressed
+#compress
+
+# packages drop log rotation information into this directory
+include /etc/logrotate.d
+
+# no packages own wtmp, or btmp -- we'll rotate them here
+/var/log/wtmp {
+    missingok
+    size 100k
+    create 0664 root utmp
+    rotate 1
+}
+
+/var/log/btmp {
+    missingok
+    size 100k
+    create 0660 root utmp
+    rotate 1
+}
+
+# system-specific logs may be configured here

files/image_config/logrotate/logrotate.d/apt

@@ -0,0 +1,15 @@
+/var/log/apt/term.log {
+  size 100k
+  rotate 1
+  compress
+  missingok
+  notifempty
+}
+
+/var/log/apt/history.log {
+  size 100k
+  rotate 1
+  compress
+  missingok
+  notifempty
+}

files/image_config/logrotate/logrotate.d/dpkg

@@ -0,0 +1,18 @@
+/var/log/dpkg.log {
+        size 100k
+        rotate 1
+        compress
+        delaycompress
+        missingok
+        notifempty
+        create 644 root root
+}
+/var/log/alternatives.log {
+        size 100k
+        rotate 1
+        compress
+        delaycompress
+        missingok
+        notifempty
+        create 644 root root
+}

files/image_config/logrotate/logrotate.d/monit

@@ -0,0 +1,12 @@
+/var/log/monit.log {
+        size 100k
+        rotate 1
+        missingok
+        create 640 root adm
+        notifempty
+        compress
+        delaycompress
+        postrotate
+                invoke-rc.d monit reload > /dev/null
+        endscript
+}

files/image_config/logrotate/logrotate.d/rsyslog

@@ -0,0 +1,93 @@
+# These logs should no longer get created. However, in case they do get created,
+# we should keep them to a small size and rotate them also.
+/var/log/mail.info
+/var/log/mail.warn
+/var/log/mail.err
+/var/log/mail.log
+/var/log/daemon.log
+/var/log/kern.log
+/var/log/user.log
+/var/log/lpr.log
+/var/log/debug
+/var/log/messages
+{
+    size 10k
+    rotate 1
+    missingok
+    notifempty
+    compress
+    delaycompress
+    sharedscripts
+    postrotate
+        # calling kill directly instead of 'service rsyslog rotate >/dev/null' due
+        # to bug in init-system-helpers. bug has apparently been fixed in v1.47.
+        # however, debian jessie is still using v1.22.
+        # see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=672218
+        kill -hup $(cat /var/run/rsyslogd.pid)
+    endscript
+}
+
+/var/log/auth.log
+/var/log/cron.log
+/var/log/syslog
+/var/log/teamd.log
+/var/log/quagga/bgpd.log
+/var/log/quagga/zebra.log
+/var/log/swss/sairedis.rec
+/var/log/swss/swss.rec
+{
+    size 512k
+    rotate 5000
+    missingok
+    notifempty
+    compress
+    delaycompress
+    nosharedscripts
+    prerotate
+        # Adjust NUM_LOGS_TO_ROTATE to reflect number of log files that trigger this block specified above
+        NUM_LOGS_TO_ROTATE=8
+
+        # Reserve space for btmp, wtmp, dpkg.log, monit.log, etc., as well as logs that
+        # should be disabled, just in case they get created and rotated
+        RESERVED_SPACE_KB=4096
+
+        VAR_LOG_SIZE_KB=$(df -k /var/log | sed -n 2p | awk '{print $2}')
+
+        # Limit usable space to 95% of the partition minus the reserved space for other logs
+        USABLE_SPACE_KB=$(( (VAR_LOG_SIZE_KB * 95 / 100) - RESERVED_SPACE_KB))
+
+        # Set our threshold so as to maintain enough space to write all logs from empty to full
+        # Most likely, some logs will have non-zero size when this is called, so this errs on the side
+        # of caution, giving us a bit of a cushion if a log grows quickly and passes its rotation size
+        THRESHOLD_KB=$((USABLE_SPACE_KB - (512 * NUM_LOGS_TO_ROTATE * 2)))
+
+        while true; do
+            USED_KB=$(du -s /var/log | awk '{ print $1; }')
+
+            if [ $USED_KB -lt $THRESHOLD_KB ]; then
+                break
+            else
+                OLDEST_ARCHIVE_FILE=$(find /var/log -type f -printf '%T+ %p\n' | grep -E '.+\.[0-9]+(\.gz)?$' | sort | head -n 1 | awk '{ print $2; }')
+
+                if [ -z "$OLDEST_ARCHIVE_FILE" ]; then
+                    logger -p syslog.err -t "logrotate" "No archive file to delete -- potential for filling up /var/log partition!"
+                    break
+                fi
+
+                logger -p syslog.info -t "logrotate" "Deleting archive file $OLDEST_ARCHIVE_FILE to free up space"
+                rm -rf "$OLDEST_ARCHIVE_FILE"
+            fi
+        done
+    endscript
+    postrotate
+        if [ $(echo $1 | grep -c "/var/log/swss/") -gt 0 ]; then
+            pgrep -x orchagent | xargs /bin/kill -HUP 2>/dev/null || true
+        else
+            # Calling kill directly instead of 'service rsyslog rotate >/dev/null' due
+            # to bug in init-system-helpers. Bug has apparently been fixed in v1.47.
+            # However, Debian Jessie is still using v1.22.
+            # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=672218
+            kill -HUP $(cat /var/run/rsyslogd.pid)
+        fi
+    endscript
+}

files/image_config/rsyslog/rsyslog.d/99-default.conf

@@ -1,61 +1,18 @@
 #
 # First some standard log files.  Log by facility.
 #
+
+# Log all facilities to /var/log/syslog except cron, auth
+# and authpriv. They are noisy - log them to their own files
+*.*;cron,auth,authpriv.none     -/var/log/syslog
 auth,authpriv.*                 /var/log/auth.log
-*.*;auth,authpriv.none          -/var/log/syslog
 cron.*                          /var/log/cron.log
-# Do not redirect daemon, kernel or lpr logs to
-# their own files. Let them log to /var/log/syslog
-#daemon.*                        -/var/log/daemon.log
-#kern.*                          -/var/log/kern.log
-#kern.*                          -/var/persist/log/kern.log
-#lpr.*                           -/var/log/lpr.log
-mail.*                          -/var/log/mail.log
-user.*                          -/var/log/user.log
-
-#
-# Logging for the mail system.  Split it up so that
-# it is easy to write scripts to parse these files.
-#
-mail.info                       -/var/log/mail.info
-mail.warn                       -/var/log/mail.warn
-mail.err                        /var/log/mail.err
-
-#
-# Logging for INN news system.
-#
-news.crit                       /var/log/news/news.crit
-news.err                        /var/log/news/news.err
-news.notice                     -/var/log/news/news.notice
 
-#
-# Some "catch-all" log files.
-#
-*.=debug;\
-        auth,authpriv.none;\
-        news.none;mail.none     -/var/log/debug
-#
-# Removed as duplicates:
-#*.=info;*.=notice;*.=warn;\
-#        auth,authpriv.none;\
-#        cron,daemon.none;\
-#        mail,news.none          -/var/log/messages
-#
-*.=crit;*.=alert;*.=emerg     -/var/persist/log/alarms
 #
 # Emergencies are sent to everybody logged in.
 #
 *.emerg                         :omusrmsg:*
 
-#
-# I like to have messages displayed on the console, but only on a virtual
-# console I usually leave idle.
-#
-#daemon,mail.*;\
-#       news.=crit;news.=err;news.=notice;\
-#       *.=debug;*.=info;\
-#       *.=notice;*.=warn       /dev/tty8
-
 # The named pipe /dev/xconsole is for the `xconsole' utility.  To use it,
 # you must invoke `xconsole' with the `-file' option:
 #