diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go
index cbbd6ae5a690..c6a797883678 100644
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go
@@ -663,6 +663,9 @@ func (ss *scaleSet) listScaleSetVMs(scaleSetName, resourceGroup string) ([]compu
 	allVMs, rerr := ss.VirtualMachineScaleSetVMsClient.List(ctx, resourceGroup, scaleSetName, string(compute.InstanceView))
 	if rerr != nil {
 		klog.Errorf("VirtualMachineScaleSetVMsClient.List failed: %v", rerr)
+		if rerr.IsNotFound() {
+			return nil, cloudprovider.InstanceNotFound
+		}
 		return nil, rerr.Error()
 	}
 
diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error.go b/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error.go
index e49cdef13f58..fb7a7bab8a87 100644
--- a/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error.go
+++ b/staging/src/k8s.io/legacy-cloud-providers/azure/retry/azure_error.go
@@ -89,6 +89,15 @@ func (err *Error) IsThrottled() bool {
 	return err.HTTPStatusCode == http.StatusTooManyRequests || err.RetryAfter.After(now())
 }
 
+// IsNotFound returns true the if the requested object wasn't found
+func (err *Error) IsNotFound() bool {
+	if err == nil {
+		return false
+	}
+
+	return err.HTTPStatusCode == http.StatusNotFound
+}
+
 // NewError creates a new Error.
 func NewError(retriable bool, err error) *Error {
 	return &Error{