feat(executor): Configurable retry backoff settings for workflow exec…

…utor (#5309)

Signed-off-by: terrytangyuan <terrytangyuan@gmail.com>

docs/environment-variables.md

@@ -38,5 +38,9 @@ Note that these environment variables may be removed at any time.
 | `ARGO_CONTAINER_RUNTIME_EXECUTOR` | `string` | The name of the container runtime executor. |
 | `ARGO_KUBELET_PORT` | `int` | The port to the Kubelet API. |
 | `ARGO_KUBELET_INSECURE` | `bool` | Whether to disable the TLS verification. |
+| `EXECUTOR_RETRY_BACKOFF_DURATION` | `time.Duration` | The retry backoff duration when the workflow executor performs retries. |
+| `EXECUTOR_RETRY_BACKOFF_FACTOR` | `float` | The retry backoff factor when the workflow executor performs retries. |
+| `EXECUTOR_RETRY_BACKOFF_JITTER` | `float` | The retry backoff jitter when the workflow executor performs retries. |
+| `EXECUTOR_RETRY_BACKOFF_STEPS` | `int` | The retry backoff steps when the workflow executor performs retries. |
 | `PNS_PRIVILEGED` | `bool` | Whether to always set privileged on for PNS when PNS executor is used. |
 | `REMOVE_LOCAL_ART_PATH` | `bool` | Whether to remove local artifacts. |

workflow/executor/executor.go

@@ -30,6 +30,7 @@ import (
 	wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
 	"github.com/argoproj/argo-workflows/v3/util"
 	"github.com/argoproj/argo-workflows/v3/util/archive"
+	envutil "github.com/argoproj/argo-workflows/v3/util/env"
 	errorsutil "github.com/argoproj/argo-workflows/v3/util/errors"
 	"github.com/argoproj/argo-workflows/v3/util/retry"
 	waitutil "github.com/argoproj/argo-workflows/v3/util/wait"
@@ -47,10 +48,10 @@ import (
 // 3	5.160
 // 4	9.256
 var ExecutorRetry = wait.Backoff{
-	Steps:    5,
-	Duration: 1 * time.Second,
-	Factor:   1.6,
-	Jitter:   0.5,
+	Steps:    envutil.LookupEnvIntOr("EXECUTOR_RETRY_BACKOFF_STEPS", 5),
+	Duration: envutil.LookupEnvDurationOr("EXECUTOR_RETRY_BACKOFF_DURATION", 1*time.Second),
+	Factor:   envutil.LookupEnvFloatOr("EXECUTOR_RETRY_BACKOFF_FACTOR", 1.6),
+	Jitter:   envutil.LookupEnvFloatOr("EXECUTOR_RETRY_BACKOFF_JITTER", 0.5),
 }
 
 const (