Merge pull request #11 from JoelSpeed/requeue-on-invalid-credentials

Bug 2030488: Requeue create on invalid credentials errors

pkg/cloud/azure/actuators/machine/actuator.go

@@ -25,6 +25,7 @@ import (
 	"github.com/Azure/go-autorest/autorest"
 	machinev1 "github.com/openshift/api/machine/v1beta1"
 	machineapierrors "github.com/openshift/machine-api-operator/pkg/controller/machine"
+	"github.com/openshift/machine-api-provider-azure/pkg/cloud/azure"
 	"github.com/openshift/machine-api-provider-azure/pkg/cloud/azure/actuators"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/record"
@@ -102,7 +103,11 @@ func (a *Actuator) Create(ctx context.Context, machine *machinev1.Machine) error
 		var detailedError autorest.DetailedError
 		if errors.As(err, &detailedError) {
 			statusCode, ok := detailedError.StatusCode.(int)
-			if ok && statusCode >= 400 && statusCode < 500 {
+			// Any 4xx error that isn't invalid credentials should be a terminal failure.
+			// Invalid Credentials implies that the credentials expired between the scope creation and API calls,
+			// this may happen when CCO is refreshing credentials simultaneously.
+			// In this case we should retry as the credentials should be updated in the secret.
+			if ok && statusCode >= 400 && statusCode < 500 && !azure.InvalidCredentials(err) {
 				return a.handleMachineError(machine, machineapierrors.InvalidMachineConfiguration("failed to reconcile machine %q: %v", machine.Name, detailedError), createEventAction)
 			}
 		}

pkg/cloud/azure/actuators/machine/actuator_test.go

@@ -768,6 +768,12 @@ func TestStatusCodeBasedCreationErrors(t *testing.T) {
 			statusCode: 300,
 			requeable:  true,
 		},
+		{
+			name:       "CreateMachine",
+			event:      "Warning FailedCreate CreateError: failed to reconcile machine \"azure-actuator-testing-machine\"s: failed to create vm azure-actuator-testing-machine: failed to create or get machine: failed to create or get machine: compute.VirtualMachinesClient#CreateOrUpdate: MOCK: StatusCode=401",
+			statusCode: 401,
+			requeable:  true,
+		},
 	}
 
 	for _, tc := range cases {

pkg/cloud/azure/errors.go

@@ -17,6 +17,8 @@ limitations under the License.
 package azure
 
 import (
+	"errors"
+
 	"github.com/Azure/go-autorest/autorest"
 )
 
@@ -27,3 +29,12 @@ func ResourceNotFound(err error) bool {
 	}
 	return false
 }
+
+// InvalidCredentials parses the error to check if its an invalid credentials error
+func InvalidCredentials(err error) bool {
+	detailedError := autorest.DetailedError{}
+	if errors.As(err, &detailedError) && detailedError.StatusCode == 401 {
+		return true
+	}
+	return false
+}