Merge pull request #2240 from k8s-infra-cherrypick-robot/cherry-pick-…

…2239-to-release-1.30 [release-1.30] fix: refine check disk lun collision logic
kubernetes-sigs · Mar 26, 2024 · 74cbf2f · 74cbf2f
2 parents 86db80a + 36e623a
commit 74cbf2f
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 7 deletions.
diff --git a/pkg/azuredisk/azuredisk.go b/pkg/azuredisk/azuredisk.go
@@ -114,6 +114,8 @@ type Driver struct {
 	volumeLocks *volumehelper.VolumeLocks
 	// a timed cache for throttling
 	throttlingCache azcache.Resource
+	// a timed cache for disk lun collision check throttling
+	checkDiskLunThrottlingCache azcache.Resource
 }
 
 // newDriverV1 Creates a NewCSIDriver object. Assumes vendor version is equal to driver version &
@@ -160,13 +162,15 @@ func newDriverV1(options *DriverOptions) *Driver {
 	}
 	topologyKey = fmt.Sprintf("topology.%s/zone", driver.Name)
 
-	cache, err := azcache.NewTimedCache(5*time.Minute, func(key string) (interface{}, error) {
-		return nil, nil
-	}, false)
-	if err != nil {
+	getter := func(key string) (interface{}, error) { return nil, nil }
+	var err error
+	if driver.throttlingCache, err = azcache.NewTimedCache(5*time.Minute, getter, false); err != nil {
 		klog.Fatalf("%v", err)
 	}
-	driver.throttlingCache = cache
+	if driver.checkDiskLunThrottlingCache, err = azcache.NewTimedCache(30*time.Minute, getter, false); err != nil {
+		klog.Fatalf("%v", err)
+	}
+
 	userAgent := GetUserAgent(driver.Name, driver.customUserAgent, driver.userAgentSuffix)
 	klog.V(2).Infof("driver userAgent: %s", userAgent)
 

diff --git a/pkg/azuredisk/controllerserver.go b/pkg/azuredisk/controllerserver.go
@@ -592,6 +592,10 @@ func (d *Driver) ValidateVolumeCapabilities(ctx context.Context, req *csi.Valida
 func (d *Driver) getOccupiedLunsFromNode(ctx context.Context, nodeName types.NodeName, diskURI string) []int {
 	var occupiedLuns []int
 	if d.checkDiskLUNCollision && !d.isCheckDiskLunThrottled() {
+		timer := time.AfterFunc(checkDiskLunThrottleLatency, func() {
+			klog.Warningf("checkDiskLun(%s) on node %s took longer than %v, disable disk lun check temporarily", diskURI, nodeName, checkDiskLunThrottleLatency)
+			d.checkDiskLunThrottlingCache.Set(consts.CheckDiskLunThrottlingKey, "")
+		})
 		now := time.Now()
 		if usedLunsFromVA, err := d.getUsedLunsFromVolumeAttachments(ctx, string(nodeName)); err == nil {
 			if len(usedLunsFromVA) > 0 {
@@ -611,9 +615,9 @@ func (d *Driver) getOccupiedLunsFromNode(ctx context.Context, nodeName types.Nod
 		}
 		latency := time.Since(now)
 		if latency > checkDiskLunThrottleLatency {
-			klog.Warningf("checkDiskLun(%s) on node %s took %v (limit: %v), disable disk lun check temporarily", diskURI, nodeName, latency, checkDiskLunThrottleLatency)
-			d.throttlingCache.Set(consts.CheckDiskLunThrottlingKey, "")
+			klog.Warningf("checkDiskLun(%s) on node %s took %v (limit: %v)", diskURI, nodeName, latency, checkDiskLunThrottleLatency)
 		} else {
+			timer.Stop() // cancel the timer
 			klog.V(6).Infof("checkDiskLun(%s) on node %s took %v", diskURI, nodeName, latency)
 		}
 	}

diff --git a/pkg/azuredisk/fake_azuredisk.go b/pkg/azuredisk/fake_azuredisk.go
@@ -139,6 +139,7 @@ func newFakeDriverV1(ctrl *gomock.Controller) (*fakeDriverV1, error) {
 		return nil, err
 	}
 	driver.throttlingCache = cache
+	driver.checkDiskLunThrottlingCache = cache
 	driver.deviceHelper = mockoptimization.NewMockInterface(ctrl)
 
 	driver.AddControllerServiceCapabilities(