diff --git a/src/cdpy/common.py b/src/cdpy/common.py index 6c4ffcc..7b64c60 100644 --- a/src/cdpy/common.py +++ b/src/cdpy/common.py @@ -198,6 +198,7 @@ def _warning_format(message, category, filename, lineno, line=None): 'DELETE_FAILED', 'Error', # DW 'installation:failed', # ML + 'provision:failed', # ML 'deprovision:failed', # ML 'BAD_HEALTH' # DF ] diff --git a/src/cdpy/datahub.py b/src/cdpy/datahub.py index 13cbc76..10cb6bb 100644 --- a/src/cdpy/datahub.py +++ b/src/cdpy/datahub.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from cdpy.common import CdpSdkBase, Squelch +from cdpy.common import CdpSdkBase, Squelch, CdpError, CdpWarning class CdpyDatahub(CdpSdkBase): @@ -28,8 +28,25 @@ def describe_all_clusters(self, environment_name=None): return [self.describe_cluster(cluster['clusterName']) for cluster in clusters_listing] return clusters_listing - def list_cluster_templates(self): - return self.sdk.call(svc='datahub', func='list_cluster_templates', ret_field='clusterTemplates') + def list_cluster_templates(self, retries=3, delay=5): + # Intermittent timeout issue in CDP 7.2.10, should be reverted to bare listing in 7.2.12 + resp = self.sdk.call( + svc='datahub', func='list_cluster_templates', ret_field='clusterTemplates', + ret_error=True + ) + if isinstance(resp, CdpError): + if retries > 0: + if str(resp.status_code) == '500' and resp.error_code == 'UNKNOWN': + retries = retries - 1 + self.sdk.throw_warning( + CdpWarning('Got likely CDP Control Plane eventual consistency error, %d retries left...' + % (retries)) + ) + self.sdk.sleep(delay) + return self.list_cluster_templates(retries, delay) + else: + self.sdk.throw_error(resp) + return resp def describe_cluster_template(self, name): return self.sdk.call(svc='datahub', func='describe_cluster_template', ret_field='clusterTemplate', squelch=[ diff --git a/src/cdpy/environments.py b/src/cdpy/environments.py index d8bd827..565d6ac 100644 --- a/src/cdpy/environments.py +++ b/src/cdpy/environments.py @@ -276,9 +276,9 @@ def create_aws_credential(self, name, role, description, retries=3, delay=2): self.sdk.throw_error(resp) return resp - def create_azure_credential(self, name, subscription, tenant, application, secret): - return self.sdk.call( - svc='environments', func='create_azure_credential', squelch=[ + def create_azure_credential(self, name, subscription, tenant, application, secret, retries=3, delay=5): + resp = self.sdk.call( + svc='environments', func='create_azure_credential', ret_error=True, squelch=[ Squelch(field='violations', value='Credential already exists with name', warning='Credential with this name already exists', default=None)], credentialName=name, @@ -286,6 +286,22 @@ def create_azure_credential(self, name, subscription, tenant, application, secre tenantId=tenant, appBased={'applicationId': application, 'secretKey': secret} ) + if isinstance(resp, CdpError): + if retries > 0: + consistency_violations = [ + 'You may have sent your authentication request to the wrong tenant' + ] + if any(x in str(resp.violations) for x in consistency_violations): + retries = retries - 1 + self.sdk.throw_warning( + CdpWarning('Got likely Azure eventual consistency error [%s], %d retries left...' + % (str(resp.violations), retries)) + ) + self.sdk.sleep(delay) + return self.create_azure_credential(name, subscription, tenant, application, secret, retries, delay) + else: + self.sdk.throw_error(resp) + return resp def create_gcp_credential(self, name, key_file): return self.sdk.call(