add network partition check in ovn probes (#1923)

dist/images/ovn-healthcheck.sh

@@ -8,5 +8,23 @@ ovn-ctl status_northd
 ovn-ctl status_ovnnb | grep -q '^running'
 ovn-ctl status_ovnsb | grep -q '^running'
 
-ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound | grep Status | grep -v "failed"
-ovn-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound | grep Status | grep -v "failed"
+nb_status=$(ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound | grep Status)
+sb_status=$(ovn-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound | grep Status)
+nb_role=$(ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound | grep Role)
+sb_role=$(ovn-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound | grep Role)
+
+if ! echo ${nb_status} | grep -v "failed"; then
+    echo "nb health check failed"
+fi
+if ! echo ${sb_status} | grep -v "failed"; then
+    echo "sb health check failed"
+fi
+
+if echo ${nb_status} | grep "disconnected" && echo ${nb_role} | grep "candidate"; then
+    echo "nb health check failed"
+    exit 1
+fi
+if echo ${sb_status} | grep "disconnected" && echo ${sb_role} | grep "candidate"; then
+    echo "sb health check failed"
+    exit 1
+fi