Various ARN handling fixes (ansible-collections#1848)

Various ARN handling fixes

Depends-On: ansible-collections#1619
SUMMARY
fixes: ansible-collections#1846
Various modules had hard-coded ARN handling which assumed the use of the main  partition.  This causes problems for folks using Gov Cloud (and aws-cn)
ISSUE TYPE

Bugfix Pull Request

COMPONENT NAME
plugins/modules/batch_compute_environment.py
plugins/modules/ec2_launch_template.py
plugins/modules/elasticache_info.py
plugins/modules/iam_group.py
plugins/modules/iam_role.py
plugins/modules/msk_config.py
plugins/modules/redshift.py
plugins/modules/sns_topic.py
ADDITIONAL INFORMATION

Reviewed-by: Alina Buzachis

batch_compute_environment.py

@@ -233,6 +233,7 @@
 from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict
 
+from ansible_collections.amazon.aws.plugins.module_utils.arn import validate_aws_arn
 from ansible_collections.community.aws.plugins.module_utils.modules import AnsibleCommunityAWSModule as AnsibleAWSModule
 
 
@@ -270,7 +271,7 @@ def validate_params(module):
         module.fail_json(
             msg=f"Function compute_environment_name {compute_environment_name} is invalid. Names must contain only alphanumeric characters and underscores."
         )
-    if not compute_environment_name.startswith("arn:aws:batch:"):
+    if not validate_aws_arn(compute_environment_name, service="batch"):
         if len(compute_environment_name) > 128:
             module.fail_json(msg=f'compute_environment_name "{compute_environment_name}" exceeds 128 character limit')
 

ec2_launch_template.py

@@ -422,7 +422,6 @@
   type: int
 """
 
-import re
 from uuid import uuid4
 
 try:
@@ -436,6 +435,7 @@
 from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict
 
+from ansible_collections.amazon.aws.plugins.module_utils.arn import validate_aws_arn
 from ansible_collections.amazon.aws.plugins.module_utils.botocore import is_boto3_error_code
 from ansible_collections.amazon.aws.plugins.module_utils.retries import AWSRetry
 from ansible_collections.amazon.aws.plugins.module_utils.tagging import ansible_dict_to_boto3_tag_list
@@ -446,7 +446,7 @@
 
 
 def determine_iam_role(module, name_or_arn):
-    if re.match(r"^arn:aws:iam::\d+:instance-profile/[\w+=/,.@-]+$", name_or_arn):
+    if validate_aws_arn(name_or_arn, service="iam", resource_type="instance-profile"):
         return {"arn": name_or_arn}
     iam = module.client("iam", retry_decorator=AWSRetry.jittered_backoff())
     try:

elasticache_info.py

@@ -410,6 +410,7 @@
 from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
 from ansible_collections.amazon.aws.plugins.module_utils.botocore import is_boto3_error_code
+from ansible_collections.amazon.aws.plugins.module_utils.iam import get_aws_account_info
 from ansible_collections.amazon.aws.plugins.module_utils.retries import AWSRetry
 from ansible_collections.amazon.aws.plugins.module_utils.tagging import boto3_tag_list_to_ansible_dict
 
@@ -444,30 +445,18 @@ def get_elasticache_tags_with_backoff(client, cluster_id):
     return client.list_tags_for_resource(ResourceName=cluster_id)["TagList"]
 
 
-def get_aws_account_id(module):
-    try:
-        client = module.client("sts")
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-        module.fail_json_aws(e, msg="Can't authorize connection")
-
-    try:
-        return client.get_caller_identity()["Account"]
-    except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-        module.fail_json_aws(e, msg="Couldn't obtain AWS account id")
-
-
 def get_elasticache_clusters(client, module):
     region = module.region
     try:
         clusters = describe_cache_clusters_with_backoff(client, cluster_id=module.params.get("name"))
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Couldn't obtain cache cluster info")
 
-    account_id = get_aws_account_id(module)
+    account_id, partition = get_aws_account_info(module)
     results = []
     for cluster in clusters:
         cluster = camel_dict_to_snake_dict(cluster)
-        arn = f"arn:aws:elasticache:{region}:{account_id}:cluster:{cluster['cache_cluster_id']}"
+        arn = f"arn:{partition}:elasticache:{region}:{account_id}:cluster:{cluster['cache_cluster_id']}"
         try:
             tags = get_elasticache_tags_with_backoff(client, arn)
         except is_boto3_error_code("CacheClusterNotFound"):

iam_group.py

@@ -171,6 +171,7 @@
 
 from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
+from ansible_collections.amazon.aws.plugins.module_utils.arn import validate_aws_arn
 from ansible_collections.amazon.aws.plugins.module_utils.botocore import is_boto3_error_code
 from ansible_collections.amazon.aws.plugins.module_utils.retries import AWSRetry
 
@@ -203,7 +204,7 @@ def compare_group_members(current_group_members, new_group_members):
 
 
 def convert_friendly_names_to_arns(connection, module, policy_names):
-    if not any(not policy.startswith("arn:") for policy in policy_names if policy is not None):
+    if all(validate_aws_arn(policy, service="iam") for policy in policy_names if policy is not None):
         return policy_names
     allpolicies = {}
     paginator = connection.get_paginator("list_policies")
@@ -213,7 +214,7 @@ def convert_friendly_names_to_arns(connection, module, policy_names):
         allpolicies[policy["PolicyName"]] = policy["Arn"]
         allpolicies[policy["Arn"]] = policy["Arn"]
     try:
-        return [allpolicies[policy] for policy in policy_names]
+        return [allpolicies[policy] for policy in policy_names if policy is not None]
     except KeyError as e:
         module.fail_json(msg="Couldn't find policy: " + str(e))
 

iam_role.py

@@ -224,6 +224,7 @@
 
 from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
+from ansible_collections.amazon.aws.plugins.module_utils.arn import validate_aws_arn
 from ansible_collections.amazon.aws.plugins.module_utils.botocore import is_boto3_error_code
 from ansible_collections.amazon.aws.plugins.module_utils.policy import compare_policies
 from ansible_collections.amazon.aws.plugins.module_utils.retries import AWSRetry
@@ -265,7 +266,7 @@ def wait_iam_exists(module, client):
 
 
 def convert_friendly_names_to_arns(module, client, policy_names):
-    if not any(not policy.startswith("arn:") for policy in policy_names):
+    if all(validate_aws_arn(policy, service="iam") for policy in policy_names if policy is not None):
         return policy_names
 
     allpolicies = {}
@@ -275,7 +276,7 @@ def convert_friendly_names_to_arns(module, client, policy_names):
         allpolicies[policy["PolicyName"]] = policy["Arn"]
         allpolicies[policy["Arn"]] = policy["Arn"]
     try:
-        return [allpolicies[policy] for policy in policy_names]
+        return [allpolicies[policy] for policy in policy_names if policy is not None]
     except KeyError as e:
         module.fail_json_aws(e, msg="Couldn't find policy")
 
@@ -746,7 +747,7 @@ def main():
     if module.params.get("boundary"):
         if module.params.get("create_instance_profile"):
             module.fail_json(msg="When using a boundary policy, `create_instance_profile` must be set to `false`.")
-        if not module.params.get("boundary").startswith("arn:aws:iam"):
+        if not validate_aws_arn(module.params.get("boundary"), service="iam"):
             module.fail_json(msg="Boundary policy must be an ARN")
     if module.params.get("max_session_duration"):
         max_session_duration = module.params.get("max_session_duration")

msk_config.py

@@ -98,6 +98,7 @@
 
 from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
 
+from ansible_collections.amazon.aws.plugins.module_utils.iam import get_aws_account_info
 from ansible_collections.amazon.aws.plugins.module_utils.retries import AWSRetry
 
 from ansible_collections.community.aws.plugins.module_utils.modules import AnsibleCommunityAWSModule as AnsibleAWSModule
@@ -283,7 +284,8 @@ def main():
     # return some useless staff in check mode if configuration doesn't exists
     # can be useful when these options are referenced by other modules during check mode run
     if module.check_mode and not response.get("Arn"):
-        arn = "arn:aws:kafka:region:account:configuration/name/id"
+        account_id, partition = get_aws_account_info(module)
+        arn = f"arn:{partition}:kafka:{module.region}:{account_id}:configuration/{module.params['name']}/id"
         revision = 1
         server_properties = ""
     else:

redshift.py

@@ -263,7 +263,7 @@
 from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict
 
 from ansible_collections.amazon.aws.plugins.module_utils.botocore import is_boto3_error_code
-from ansible_collections.amazon.aws.plugins.module_utils.iam import get_aws_account_id
+from ansible_collections.amazon.aws.plugins.module_utils.iam import get_aws_account_info
 from ansible_collections.amazon.aws.plugins.module_utils.retries import AWSRetry
 from ansible_collections.amazon.aws.plugins.module_utils.tagging import ansible_dict_to_boto3_tag_list
 from ansible_collections.amazon.aws.plugins.module_utils.tagging import boto3_tag_list_to_ansible_dict
@@ -275,9 +275,9 @@
 def _ensure_tags(redshift, identifier, existing_tags, module):
     """Compares and update resource tags"""
 
-    account_id = get_aws_account_id(module)
-    region = module.params.get("region")
-    resource_arn = f"arn:aws:redshift:{region}:{account_id}:cluster:{identifier}"
+    account_id, partition = get_aws_account_info(module)
+    region = module.region
+    resource_arn = f"arn:{partition}:redshift:{region}:{account_id}:cluster:{identifier}"
     tags = module.params.get("tags")
     purge_tags = module.params.get("purge_tags")
 

sns_topic.py

@@ -338,9 +338,10 @@
 except ImportError:
     pass  # handled by AnsibleAWSModule
 
-from ansible_collections.amazon.aws.plugins.module_utils.transformation import scrub_none_parameters
+from ansible_collections.amazon.aws.plugins.module_utils.arn import parse_aws_arn
 from ansible_collections.amazon.aws.plugins.module_utils.policy import compare_policies
 from ansible_collections.amazon.aws.plugins.module_utils.tagging import ansible_dict_to_boto3_tag_list
+from ansible_collections.amazon.aws.plugins.module_utils.transformation import scrub_none_parameters
 
 from ansible_collections.community.aws.plugins.module_utils.modules import AnsibleCommunityAWSModule as AnsibleAWSModule
 from ansible_collections.community.aws.plugins.module_utils.sns import list_topics
@@ -579,7 +580,7 @@ def _delete_topic(self):
         return True
 
     def _name_is_arn(self):
-        return self.name.startswith("arn:")
+        return bool(parse_aws_arn(self.name))
 
     def ensure_ok(self):
         changed = False