Bug 1985033: Make inactivity_probe configurable

Making inactivity_probe configurable. Changing default inactivity_probe from 60 to 180 seconds.
openshift · Jul 27, 2021 · e99952b · e99952b
1 parent 8586629
commit e99952b
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 6 deletions.
diff --git a/bindata/network/ovn-kubernetes/ovnkube-master.yaml b/bindata/network/ovn-kubernetes/ovnkube-master.yaml
@@ -248,9 +248,9 @@ spec:
                 if [[ "${K8S_NODE_IP}" == "${CLUSTER_INITIATOR_IP}" ]]; then
                   echo "$(date -Iseconds) - nbdb - postStart - waiting for master to be selected"
 
-                  # set the connection and disable inactivity probe
+                  # set the connection and inactivity probe
                   retries=0
-                  while ! ovn-nbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_NB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe=60000; do
+                  while ! ovn-nbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_NB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe={{.OVN_NB_INACTIVITY_PROBE}}; do
                     (( retries += 1 ))
                   if [[ "${retries}" -gt 40 ]]; then
                     echo "$(date -Iseconds) - ERROR RESTARTING - nbdb - too many failed ovn-nbctl attempts, giving up"
@@ -622,9 +622,9 @@ spec:
                 if [[ "${K8S_NODE_IP}" == "${CLUSTER_INITIATOR_IP}" ]]; then
                   echo "$(date -Iseconds) - sdb - postStart - waiting for master to be selected"
 
-                  # set the connection and disable inactivity probe
+                  # set the connection and inactivity probe
                   retries=0
-                  while ! ovn-sbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_SB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe=60000; do
+                  while ! ovn-sbctl --no-leader-only -t 5 set-connection pssl:{{.OVN_SB_PORT}}{{.LISTEN_DUAL_STACK}} -- set connection . inactivity_probe={{.OVN_CONTROLLER_INACTIVITY_PROBE}}; do
                     (( retries += 1 ))
                   if [[ "${retries}" -gt 40 ]]; then
                     echo "$(date -Iseconds) - ERROR RESTARTING - sbdb - too many failed ovn-sbctl attempts, giving up"

diff --git a/manifests/0000_70_cluster-network-operator_03_deployment-ibm-cloud-managed.yaml b/manifests/0000_70_cluster-network-operator_03_deployment-ibm-cloud-managed.yaml
@@ -62,7 +62,9 @@ spec:
         - name: OVN_NORTHD_PROBE_INTERVAL
           value: "5000"
         - name: OVN_CONTROLLER_INACTIVITY_PROBE
-          value: "30000"
+          value: "180000"
+        - name: OVN_NB_INACTIVITY_PROBE
+          value: "180000"
         - name: EGRESS_ROUTER_CNI_IMAGE
           value: quay.io/openshift/origin-egress-router-cni:latest
         - name: KURYR_DAEMON_IMAGE

diff --git a/manifests/0000_70_cluster-network-operator_03_deployment.yaml b/manifests/0000_70_cluster-network-operator_03_deployment.yaml
@@ -68,7 +68,9 @@ spec:
         - name: OVN_NORTHD_PROBE_INTERVAL
           value: "5000"
         - name: OVN_CONTROLLER_INACTIVITY_PROBE
-          value: "30000"
+          value: "180000"
+        - name: OVN_NB_INACTIVITY_PROBE
+          value: "180000"
         - name: EGRESS_ROUTER_CNI_IMAGE
           value: "quay.io/openshift/origin-egress-router-cni:latest"
         - name: KURYR_DAEMON_IMAGE

diff --git a/pkg/network/ovn_kubernetes.go b/pkg/network/ovn_kubernetes.go
@@ -79,6 +79,18 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 	data.Data["OVN_NB_RAFT_ELECTION_TIMER"] = os.Getenv("OVN_NB_RAFT_ELECTION_TIMER")
 	data.Data["OVN_SB_RAFT_ELECTION_TIMER"] = os.Getenv("OVN_SB_RAFT_ELECTION_TIMER")
 	data.Data["OVN_CONTROLLER_INACTIVITY_PROBE"] = os.Getenv("OVN_CONTROLLER_INACTIVITY_PROBE")
+	controller_inactivity_probe := os.Getenv("OVN_CONTROLLER_INACTIVITY_PROBE")
+	if len(controller_inactivity_probe) == 0 {
+		controller_inactivity_probe = "180000"
+		klog.Infof("OVN_CONTROLLER_INACTIVITY_PROBE env var is not defined. Using: %s", controller_inactivity_probe)
+	}
+	data.Data["OVN_CONTROLLER_INACTIVITY_PROBE"] = controller_inactivity_probe
+	nb_inactivity_probe := os.Getenv("OVN_NB_INACTIVITY_PROBE")
+	if len(nb_inactivity_probe) == 0 {
+		nb_inactivity_probe = "180000"
+		klog.Infof("OVN_NB_INACTIVITY_PROBE env var is not defined. Using: %s", nb_inactivity_probe)
+	}
+	data.Data["OVN_NB_INACTIVITY_PROBE"] = nb_inactivity_probe
 	data.Data["OVN_NB_DB_LIST"] = dbList(bootstrapResult.OVN.MasterIPs, OVN_NB_PORT)
 	data.Data["OVN_SB_DB_LIST"] = dbList(bootstrapResult.OVN.MasterIPs, OVN_SB_PORT)
 	data.Data["OVN_DB_CLUSTER_INITIATOR"] = bootstrapResult.OVN.ClusterInitiator