feat(controller): Make MAX_OPERATION_TIME configurable. Close #4239 (#…

…4562)

Signed-off-by: Alex Collins <alex_collins@intuit.com>

util/env/env.go

@@ -0,0 +1,21 @@
+package env
+
+import (
+	"os"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func LookupEnvDurationOr(key string, o time.Duration) time.Duration {
+	v, found := os.LookupEnv(key)
+	if found {
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			log.WithField(key, v).WithError(err).Panic("failed to parse")
+		} else {
+			return d
+		}
+	}
+	return o
+}

util/env/env_test.go

@@ -0,0 +1,18 @@
+package env
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLookupEnvDurationOr(t *testing.T) {
+	defer func() { _ = os.Unsetenv("FOO") }()
+	assert.Equal(t, time.Second, LookupEnvDurationOr("", time.Second), "default value")
+	_ = os.Setenv("FOO", "bar")
+	assert.Panics(t, func() { LookupEnvDurationOr("FOO", time.Second) }, "bad value")
+	_ = os.Setenv("FOO", "1h")
+	assert.Equal(t, time.Hour, LookupEnvDurationOr("FOO", time.Second), "env var value")
+}

workflow/controller/operator.go

@@ -37,6 +37,7 @@ import (
 	wfv1 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
 	"github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
 	"github.com/argoproj/argo/util"
+	envutil "github.com/argoproj/argo/util/env"
 	errorsutil "github.com/argoproj/argo/util/errors"
 	"github.com/argoproj/argo/util/intstr"
 	"github.com/argoproj/argo/util/resource"
@@ -116,8 +117,10 @@ var (
 
 // maxOperationTime is the maximum time a workflow operation is allowed to run
 // for before requeuing the workflow onto the workqueue.
-const maxOperationTime = 10 * time.Second
-const defaultRequeueTime = maxOperationTime
+var (
+	maxOperationTime   = envutil.LookupEnvDurationOr("MAX_OPERATION_TIME", 30*time.Second)
+	defaultRequeueTime = envutil.LookupEnvDurationOr("DEFAULT_REQUEUE_TIME", maxOperationTime/2)
+)
 
 // failedNodeStatus is a subset of NodeStatus that is only used to Marshal certain fields into a JSON of failed nodes
 type failedNodeStatus struct {