Added retry around RuntimeExecutor.Wait call when waiting for main container completion (argoproj#1597)

Author: markterm

workflow/executor/executor.go

@@ -888,7 +888,14 @@ func (we *WorkflowExecutor) Wait() error {
 	annotationUpdatesCh := we.monitorAnnotations(ctx)
 	go we.monitorDeadline(ctx, annotationUpdatesCh)
 
-	err = we.RuntimeExecutor.Wait(mainContainerID)
+	_ = wait.ExponentialBackoff(retry.DefaultRetry, func() (bool, error) {
+		err = we.RuntimeExecutor.Wait(mainContainerID)
+		if err != nil {
+			log.Warnf("Failed to wait for container id '%s': %v", mainContainerID, err)
+			return false, err
+		}
+		return true, nil
+	})
 	if err != nil {
 		return err
 	}