Retry all OVS executions, not only transactions

pkg/network/node/ovs/ovs.go

@@ -106,9 +106,14 @@ type Transaction interface {
 const (
 	OVS_OFCTL = "ovs-ofctl"
 	OVS_VSCTL = "ovs-vsctl"
-	RETRIES   = 10
 )
 
+var ovsBackoff utilwait.Backoff = utilwait.Backoff{
+	Duration: 500 * time.Millisecond,
+	Factor:   1.25,
+	Steps:    10,
+}
+
 // ovsExec implements ovs.Interface via calls to ovs-ofctl and ovs-vsctl
 type ovsExec struct {
 	execer exec.Interface
@@ -189,7 +194,17 @@ func (ovsif *ovsExec) execWithStdin(cmd string, stdinArgs []string, args ...stri
 }
 
 func (ovsif *ovsExec) exec(cmd string, args ...string) (string, error) {
-	return ovsif.execWithStdin(cmd, nil, args...)
+	var output string
+	var err error
+	return output, utilwait.ExponentialBackoff(ovsBackoff, func() (bool, error) {
+		output, err = ovsif.execWithStdin(cmd, nil, args...)
+		if err == nil {
+			metrics.OVSOperationsResult.WithLabelValues(metrics.OVSOperationSuccess).Inc()
+			return true, nil
+		}
+		metrics.OVSOperationsResult.WithLabelValues(metrics.OVSOperationFailure).Inc()
+		return false, nil
+	})
 }
 
 func validateColumns(columns ...string) error {
@@ -443,12 +458,7 @@ func (tx *ovsExecTx) Commit() error {
 	defer func() {
 		tx.flows = []string{}
 	}()
-	backoff := utilwait.Backoff{
-		Duration: 500 * time.Millisecond,
-		Factor:   1.25,
-		Steps:    RETRIES,
-	}
-	return utilwait.ExponentialBackoff(backoff, func() (bool, error) {
+	return utilwait.ExponentialBackoff(ovsBackoff, func() (bool, error) {
 		err := tx.ovsif.bundle(tx.flows)
 		if err == nil {
 			metrics.OVSOperationsResult.WithLabelValues(metrics.OVSOperationSuccess).Inc()

pkg/network/node/ovs/ovs_test.go

@@ -109,7 +109,7 @@ func TestTransactionSuccess(t *testing.T) {
 	ensureTestResults(t, fexec)
 
 	// Test Failed transaction
-	for i := 0; i < RETRIES; i++ {
+	for i := 0; i < ovsBackoff.Steps; i++ {
 		fakeCmd = addTestResult(t, fexec, "ovs-ofctl -O OpenFlow13 bundle br0 -", "", fmt.Errorf("Something bad happened"))
 	}
 	otx = ovsif.NewTransaction()
@@ -126,7 +126,7 @@ func TestTransactionSuccess(t *testing.T) {
 	ensureInputFlows(t, fakeCmd, expectedInputFlows)
 
 	// Test a couple of failed transaction, but that finally pass
-	for i := 0; i < RETRIES-1; i++ {
+	for i := 0; i < ovsBackoff.Steps-1; i++ {
 		fakeCmd = addTestResult(t, fexec, "ovs-ofctl -O OpenFlow13 bundle br0 -", "", fmt.Errorf("Something bad happened"))
 	}
 	fakeCmd = addTestResult(t, fexec, "ovs-ofctl -O OpenFlow13 bundle br0 -", "", nil)