Merge pull request #1369 from mandre/haproxy_ps

Bug 1771566: [OpenStack] verify that 'old' HAProxy processes being deleted after HAProxy reload

templates/master/00-master/openstack/files/openstack-haproxy.yaml

@@ -31,16 +31,35 @@ contents:
       containers:
       - name: haproxy
         image: {{.Images.haproxyImage}}
+        env:
+          - name: OLD_HAPROXY_PS_FORCE_DEL_TIMEOUT
+            value: "120"
         command:
         - "/bin/bash"
         - "-c"
         - |
           #/bin/bash
+          verify_old_haproxy_ps_being_deleted()
+          {
+            local prev_pids
+            prev_pids="$1"
+            sleep $OLD_HAPROXY_PS_FORCE_DEL_TIMEOUT
+            cur_pids=$(pidof haproxy)
+            for val in $prev_pids; do
+                if [[ $cur_pids =~ (^|[[:space:]])"$val"($|[[:space:]]) ]] ; then
+                   kill $val
+                fi
+            done
+          }
+
           reload_haproxy()
           {
             old_pids=$(pidof haproxy)
             if [ -n "$old_pids" ]; then
                 /usr/sbin/haproxy -W -db -f /etc/haproxy/haproxy.cfg  -p /var/lib/haproxy/run/haproxy.pid -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids &
+                #There seems to be some cases where HAProxy doesn't drain properly.
+                #To handle that case, SIGTERM signal being sent to old HAProxy processes which haven't terminated.
+                verify_old_haproxy_ps_being_deleted "$old_pids"  &
             else
                 /usr/sbin/haproxy -W -db -f /etc/haproxy/haproxy.cfg  -p /var/lib/haproxy/run/haproxy.pid &
             fi
@@ -61,6 +80,7 @@ contents:
           declare -r haproxy_log_sock="/var/run/haproxy/haproxy-log.sock"
           export -f msg_handler
           export -f reload_haproxy
+          export -f verify_old_haproxy_ps_being_deleted
           rm -f "$haproxy_sock" "$haproxy_log_sock"
           socat UNIX-RECV:${haproxy_log_sock} STDOUT &
           if [ -s "/etc/haproxy/haproxy.cfg" ]; then