Make the maximum polling backoff configurable (openshift#1607)

cmd/controller-manager/app/controller_manager.go

@@ -332,6 +332,7 @@ func StartControllers(s *options.ControllerManagerServer,
 			s.OSBAPIPreferredVersion,
 			recorder,
 			s.ReconciliationRetryDuration,
+			s.OperationPollingMaximumBackoffDuration,
 		)
 		if err != nil {
 			return err

cmd/controller-manager/app/options/options.go

@@ -39,17 +39,18 @@ type ControllerManagerServer struct {
 }
 
 const (
-	defaultResyncInterval               = 5 * time.Minute
-	defaultServiceBrokerRelistInterval  = 24 * time.Hour
-	defaultContentType                  = "application/json"
-	defaultBindAddress                  = "0.0.0.0"
-	defaultPort                         = 10000
-	defaultK8sKubeconfigPath            = "./kubeconfig"
-	defaultServiceCatalogKubeconfigPath = "./service-catalog-kubeconfig"
-	defaultOSBAPIContextProfile         = true
-	defaultConcurrentSyncs              = 5
-	defaultLeaderElectionNamespace      = "kube-system"
-	defaultReconciliationRetryDuration  = 7 * 24 * time.Hour
+	defaultResyncInterval                         = 5 * time.Minute
+	defaultServiceBrokerRelistInterval            = 24 * time.Hour
+	defaultContentType                            = "application/json"
+	defaultBindAddress                            = "0.0.0.0"
+	defaultPort                                   = 10000
+	defaultK8sKubeconfigPath                      = "./kubeconfig"
+	defaultServiceCatalogKubeconfigPath           = "./service-catalog-kubeconfig"
+	defaultOSBAPIContextProfile                   = true
+	defaultConcurrentSyncs                        = 5
+	defaultLeaderElectionNamespace                = "kube-system"
+	defaultReconciliationRetryDuration            = 7 * 24 * time.Hour
+	defaultOperationPollingMaximumBackoffDuration = 20 * time.Minute
 )
 
 var defaultOSBAPIPreferredVersion = osb.LatestAPIVersion().HeaderValue()
@@ -59,21 +60,22 @@ var defaultOSBAPIPreferredVersion = osb.LatestAPIVersion().HeaderValue()
 func NewControllerManagerServer() *ControllerManagerServer {
 	s := ControllerManagerServer{
 		ControllerManagerConfiguration: componentconfig.ControllerManagerConfiguration{
-			Address:                      defaultBindAddress,
-			Port:                         defaultPort,
-			ContentType:                  defaultContentType,
-			K8sKubeconfigPath:            defaultK8sKubeconfigPath,
-			ServiceCatalogKubeconfigPath: defaultServiceCatalogKubeconfigPath,
-			ResyncInterval:               defaultResyncInterval,
-			ServiceBrokerRelistInterval:  defaultServiceBrokerRelistInterval,
-			OSBAPIContextProfile:         defaultOSBAPIContextProfile,
-			OSBAPIPreferredVersion:       defaultOSBAPIPreferredVersion,
-			ConcurrentSyncs:              defaultConcurrentSyncs,
-			LeaderElection:               leaderelectionconfig.DefaultLeaderElectionConfiguration(),
-			LeaderElectionNamespace:      defaultLeaderElectionNamespace,
-			EnableProfiling:              true,
-			EnableContentionProfiling:    false,
-			ReconciliationRetryDuration:  defaultReconciliationRetryDuration,
+			Address:                                defaultBindAddress,
+			Port:                                   defaultPort,
+			ContentType:                            defaultContentType,
+			K8sKubeconfigPath:                      defaultK8sKubeconfigPath,
+			ServiceCatalogKubeconfigPath:           defaultServiceCatalogKubeconfigPath,
+			ResyncInterval:                         defaultResyncInterval,
+			ServiceBrokerRelistInterval:            defaultServiceBrokerRelistInterval,
+			OSBAPIContextProfile:                   defaultOSBAPIContextProfile,
+			OSBAPIPreferredVersion:                 defaultOSBAPIPreferredVersion,
+			ConcurrentSyncs:                        defaultConcurrentSyncs,
+			LeaderElection:                         leaderelectionconfig.DefaultLeaderElectionConfiguration(),
+			LeaderElectionNamespace:                defaultLeaderElectionNamespace,
+			EnableProfiling:                        true,
+			EnableContentionProfiling:              false,
+			ReconciliationRetryDuration:            defaultReconciliationRetryDuration,
+			OperationPollingMaximumBackoffDuration: defaultOperationPollingMaximumBackoffDuration,
 		},
 	}
 	s.LeaderElection.LeaderElect = true
@@ -100,6 +102,7 @@ func (s *ControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
 	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
 	fs.StringVar(&s.LeaderElectionNamespace, "leader-election-namespace", s.LeaderElectionNamespace, "Namespace to use for leader election lock")
 	fs.DurationVar(&s.ReconciliationRetryDuration, "reconciliation-retry-duration", s.ReconciliationRetryDuration, "The maximum amount of time to retry reconciliations on a resource before failing")
+	fs.DurationVar(&s.OperationPollingMaximumBackoffDuration, "operation-polling-maximum-backoff-duration", s.OperationPollingMaximumBackoffDuration, "The maximum amount of time to back-off while polling an OSB API operation")
 
 	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }

pkg/apis/componentconfig/types.go

@@ -96,4 +96,8 @@ type ControllerManagerConfiguration struct {
 	// ReconciliationRetryDuration is the longest time to attempt reconciliation
 	// on a given resource before failing the reconciliation
 	ReconciliationRetryDuration time.Duration
+
+	// OperationPollingMaximumBackoffDuration is the maximum duration that exponential
+	// backoff for polling OSB API operations will use.
+	OperationPollingMaximumBackoffDuration time.Duration
 }

pkg/controller/controller.go

@@ -52,9 +52,8 @@ const (
 	//
 	// 5ms, 10ms, 20ms, 40ms, 80ms, 160ms, 320ms, 640ms, 1.3s, 2.6s, 5.1s, 10.2s, 20.4s, 41s, 82s
 	maxRetries = 15
-	//
-	pollingStartInterval      = 1 * time.Second
-	pollingMaxBackoffDuration = 1 * time.Hour
+	// pollingStartInterval is the initial interval to use when polling async OSB operations.
+	pollingStartInterval = 1 * time.Second
 
 	// ContextProfilePlatformKubernetes is the platform name sent in the OSB
 	// ContextProfile for requests coming from Kubernetes.
@@ -75,6 +74,7 @@ func NewController(
 	osbAPIPreferredVersion string,
 	recorder record.EventRecorder,
 	reconciliationRetryDuration time.Duration,
+	operationPollingMaximumBackoffDuration time.Duration,
 ) (Controller, error) {
 	controller := &controller{
 		kubeClient:                  kubeClient,
@@ -89,8 +89,8 @@ func NewController(
 		servicePlanQueue:            workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "service-plan"),
 		instanceQueue:               workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "service-instance"),
 		bindingQueue:                workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "service-binding"),
-		instancePollingQueue:        workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(pollingStartInterval, pollingMaxBackoffDuration), "instance-poller"),
-		bindingPollingQueue:         workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(pollingStartInterval, pollingMaxBackoffDuration), "binding-poller"),
+		instancePollingQueue:        workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(pollingStartInterval, operationPollingMaximumBackoffDuration), "instance-poller"),
+		bindingPollingQueue:         workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(pollingStartInterval, operationPollingMaximumBackoffDuration), "binding-poller"),
 	}
 
 	controller.brokerLister = brokerInformer.Lister()

pkg/controller/controller_test.go

@@ -1547,6 +1547,7 @@ func newTestController(t *testing.T, config fakeosb.FakeClientConfiguration) (
 		osb.LatestAPIVersion().HeaderValue(),
 		fakeRecorder,
 		7*24*time.Hour,
+		7*24*time.Hour,
 	)
 	if err != nil {
 		t.Fatal(err)

test/integration/controller_test.go

@@ -398,6 +398,7 @@ func newTestController(t *testing.T) (
 		osb.LatestAPIVersion().HeaderValue(),
 		fakeRecorder,
 		7*24*time.Hour,
+		7*24*time.Hour,
 	)
 	t.Log("controller start")
 	if err != nil {