Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Automated cherry pick of #84917: fix race condition when attach/delete disk #85107

Merged
merged 1 commit into from
Nov 30, 2019
Merged

Automated cherry pick of #84917: fix race condition when attach/delete disk #85107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 10 additions & 1 deletion staging/src/k8s.io/legacy-cloud-providers/azure/azure_controller_common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"fmt"
"path"
"strings"
"sync"
"time"

"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-03-01/compute"
Expand Down Expand Up @@ -63,7 +64,9 @@ type controllerCommon struct {
location string
storageEndpointSuffix string
resourceGroup string
cloud *Cloud
// store disk URI when disk is in attaching or detaching process
diskAttachDetachMap sync.Map
cloud *Cloud
}

// getNodeVMSet gets the VMSet interface based on config.VMType and the real virtual machine type.
Expand Down Expand Up @@ -143,6 +146,8 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri
}

klog.V(2).Infof("Trying to attach volume %q lun %d to node %q.", diskURI, lun, nodeName)
c.diskAttachDetachMap.Store(strings.ToLower(diskURI), "attaching")
defer c.diskAttachDetachMap.Delete(strings.ToLower(diskURI))
return lun, vmset.AttachDisk(isManagedDisk, diskName, diskURI, nodeName, lun, cachingMode)
}

Expand All @@ -169,14 +174,18 @@ func (c *controllerCommon) DetachDisk(diskName, diskURI string, nodeName types.N

// make the lock here as small as possible
diskOpMutex.LockKey(instanceid)
c.diskAttachDetachMap.Store(strings.ToLower(diskURI), "detaching")
resp, err := vmset.DetachDisk(diskName, diskURI, nodeName)
c.diskAttachDetachMap.Delete(strings.ToLower(diskURI))
diskOpMutex.UnlockKey(instanceid)

if c.cloud.CloudProviderBackoff && shouldRetryHTTPRequest(resp, err) {
klog.V(2).Infof("azureDisk - update backing off: detach disk(%s, %s), err: %v", diskName, diskURI, err)
retryErr := kwait.ExponentialBackoff(c.cloud.requestBackoff(), func() (bool, error) {
diskOpMutex.LockKey(instanceid)
c.diskAttachDetachMap.Store(strings.ToLower(diskURI), "detaching")
resp, err := vmset.DetachDisk(diskName, diskURI, nodeName)
c.diskAttachDetachMap.Delete(strings.ToLower(diskURI))
diskOpMutex.UnlockKey(instanceid)
return c.cloud.processHTTPRetryResponse(nil, "", resp, err)
})
Expand Down
4 changes: 4 additions & 0 deletions staging/src/k8s.io/legacy-cloud-providers/azure/azure_managedDiskController.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,10 @@ func (c *ManagedDiskController) DeleteManagedDisk(diskURI string) error {
ctx, cancel := getContextWithCancel()
defer cancel()

if _, ok := c.common.diskAttachDetachMap.Load(strings.ToLower(diskURI)); ok {
return fmt.Errorf("failed to delete disk(%s) since it's in attaching or detaching state", diskURI)
}

_, err = c.common.cloud.DisksClient.Delete(ctx, resourceGroup, diskName)
if err != nil {
return err
Expand Down