OpenStack: Fix keepalived dysfunction on vrrp iface change

When using CNV or other operators that modify how the node is connected
to the network, we may end up in the case where the configured VRRP
interface no longer has an address in the network that it is configured
to hold virtual IPs in.

This patch takes a page from what we do for HAProxy and adds a monitor
side car container that checks keepalived and reloads it when necessary.

This ports openshift#1124 to OpenStack platform, alongside with fixes from openshift#1508
and openshift#1604.

templates/common/openstack/files/openstack-keepalived.yaml

@@ -22,31 +22,8 @@ contents:
       - name: conf-dir
         hostPath:
           path: "/etc/keepalived"
-      initContainers:
-      - name: render-config
-        image: {{ .Images.baremetalRuntimeCfgImage }}
-        command:
-        - runtimecfg
-        - render
-        - "/etc/kubernetes/kubeconfig"
-        - "--api-vip"
-        - "{{ .Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}"
-        - "--dns-vip"
-        - "{{ .Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}"
-        - "--ingress-vip"
-        - "{{ .Infra.Status.PlatformStatus.OpenStack.IngressIP }}"
-        - "/config"
-        - "--out-dir"
-        - "/etc/keepalived"
-        resources: {}
-        volumeMounts:
-        - name: resource-dir
-          mountPath: "/config"
-        - name: kubeconfig
-          mountPath: "/etc/kubernetes/kubeconfig"
-        - name: conf-dir
-          mountPath: "/etc/keepalived"
-        imagePullPolicy: IfNotPresent
+      - name: run-dir
+        empty-dir: {}
       containers:
       - name: keepalived
         securityContext:
@@ -56,31 +33,88 @@ contents:
           - name: NSS_SDB_USE_CACHE
             value: "no"
         command:
-        - /usr/sbin/keepalived
-        args:
-        - "-f"
-        - "/etc/keepalived/keepalived.conf"
-        - "--dont-fork"
-        - "--vrrp"
-        - "--log-detail"
-        - "--log-console"
+        - /bin/bash
+        - -c
+        - |
+          #/bin/bash
+          reload_keepalived()
+          {
+            if pid=$(pgrep -o keepalived); then
+                kill -s SIGHUP "$pid"
+            else
+                /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console &
+            fi
+          }
+
+          msg_handler()
+          {
+            while read -r line; do
+              echo "The client sent: $line" >&2
+              # currently only 'reload' msg is supported
+              if [ "$line" = reload ]; then
+                  reload_keepalived
+              fi
+            done
+          }
+
+          set -ex
+          declare -r keepalived_sock="/var/run/keepalived/keepalived.sock"
+          export -f msg_handler
+          export -f reload_keepalived
+          if [ -s "/etc/keepalived/keepalived.conf" ]; then
+              /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console &
+          fi
+
+          rm -f "$keepalived_sock"
+          socat UNIX-LISTEN:${keepalived_sock},fork system:'bash -c msg_handler'
         resources:
           requests:
             cpu: 100m
             memory: 200Mi
         volumeMounts:
         - name: conf-dir
           mountPath: "/etc/keepalived"
+        - name: run-dir
+          mountPath: "/var/run/keepalived"
         livenessProbe:
           exec:
             command:
-            - pgrep
-            - keepalived
-          initialDelaySeconds: 10
+            - /bin/bash
+            - -c
+            - |
+              kill -s SIGUSR1 "$(pgrep -o keepalived)" && ! grep -q "State = FAULT" /tmp/keepalived.data
+          initialDelaySeconds: 20
         terminationMessagePolicy: FallbackToLogsOnError
         imagePullPolicy: IfNotPresent
+      - name: keepalived-monitor
+        image: {{ .Images.baremetalRuntimeCfgImage }}
+        command:
+        - dynkeepalived
+        - "/etc/kubernetes/kubeconfig"
+        - "/config/keepalived.conf.tmpl"
+        - "/etc/keepalived/keepalived.conf"
+        - "--api-vip"
+        - "{{ .Infra.Status.PlatformStatus.OpenStack.APIServerInternalIP }}"
+        - "--dns-vip"
+        - "{{ .Infra.Status.PlatformStatus.OpenStack.NodeDNSIP }}"
+        - "--ingress-vip"
+        - "{{ .Infra.Status.PlatformStatus.OpenStack.IngressIP }}"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - name: resource-dir
+          mountPath: "/config"
+        - name: kubeconfig
+          mountPath: "/etc/kubernetes/kubeconfig"
+        - name: conf-dir
+          mountPath: "/etc/keepalived"
+        - name: run-dir
+          mountPath: "/var/run/keepalived"
+        imagePullPolicy: IfNotPresent
       hostNetwork: true
       tolerations:
       - operator: Exists
       priorityClassName: system-node-critical
-    status: {}
+    status: {}