From 740e3313001d9125e12168e24c5e0ccf60ff9fdc Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Fri, 14 Jul 2023 14:17:13 -0500 Subject: [PATCH 01/11] Now passing the template based value to template_generation functions in the AWS detect handler. --- iambic/plugins/v0_1_0/aws/handlers.py | 36 +++++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/iambic/plugins/v0_1_0/aws/handlers.py b/iambic/plugins/v0_1_0/aws/handlers.py index 1ddb89d5a..ce2f98e12 100644 --- a/iambic/plugins/v0_1_0/aws/handlers.py +++ b/iambic/plugins/v0_1_0/aws/handlers.py @@ -20,7 +20,10 @@ Variable, ) from iambic.core.parser import load_templates -from iambic.core.template_generation import get_existing_template_map +from iambic.core.template_generation import ( + get_existing_template_map, + templatize_resource, +) from iambic.core.utils import async_batch_processor, gather_templates, yaml from iambic.plugins.v0_1_0.aws.event_bridge.models import ( GroupMessageDetails, @@ -69,6 +72,7 @@ from iambic.plugins.v0_1_0.aws.organizations.scp.utils import ( service_control_policy_is_enabled, ) +from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map if TYPE_CHECKING: from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig @@ -514,6 +518,7 @@ async def detect_changes( # noqa: C901 log.debug("No cloudtrail changes queue arn found. Returning") return + aws_account_map = await get_aws_account_map(config) role_messages = [] user_messages = [] group_messages = [] @@ -584,6 +589,7 @@ async def detect_changes( # noqa: C901 ) if actor != identity_arn: account_id = decoded_message.get("recipientAccountId") + aws_account = aws_account_map[account_id] request_params = decoded_message["requestParameters"] response_elements = decoded_message["responseElements"] event = decoded_message["eventName"] @@ -595,7 +601,9 @@ async def detect_changes( # noqa: C901 role_messages.append( RoleMessageDetails( account_id=account_id, - role_name=role_name, + role_name=templatize_resource( + aws_account, role_name + ), delete=bool(event == "DeleteRole"), ) ) @@ -605,7 +613,9 @@ async def detect_changes( # noqa: C901 user_messages.append( UserMessageDetails( account_id=account_id, - role_name=user_name, + user_name=templatize_resource( + aws_account, user_name + ), delete=bool(event == "DeleteUser"), ) ) @@ -615,7 +625,9 @@ async def detect_changes( # noqa: C901 group_messages.append( GroupMessageDetails( account_id=account_id, - group_name=group_name, + group_name=templatize_resource( + aws_account, group_name + ), delete=bool(event == "DeleteGroup"), ) ) @@ -632,8 +644,12 @@ async def detect_changes( # noqa: C901 managed_policy_messages.append( ManagedPolicyMessageDetails( account_id=account_id, - policy_name=policy_name, - policy_path=policy_path, + policy_name=templatize_resource( + aws_account, policy_name + ), + policy_path=templatize_resource( + aws_account, policy_path + ), delete=bool( decoded_message["eventName"] == "DeletePolicy" ), @@ -647,8 +663,12 @@ async def detect_changes( # noqa: C901 permission_set_messages.append( PermissionSetMessageDetails( account_id=account_id, - instance_arn=request_params.get("instanceArn"), - permission_set_arn=permission_set_arn, + instance_arn=templatize_resource( + aws_account, request_params.get("instanceArn") + ), + permission_set_arn=templatize_resource( + aws_account, permission_set_arn + ), ) ) elif scp_policy_id := SCPMessageDetails.get_policy_id( From 8d84d0caa00cf434c5b4aeb2532eae7377d478ad Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Fri, 14 Jul 2023 18:17:52 -0500 Subject: [PATCH 02/11] Added dedicated function for rendering a jinja string value with a provider child. New functions for aggregating detect messages. --- iambic/core/detect.py | 40 ++++++++++++++++++++++++++++++++++++++++ iambic/core/models.py | 38 ++++++++++++-------------------------- iambic/core/utils.py | 28 +++++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 27 deletions(-) create mode 100644 iambic/core/detect.py diff --git a/iambic/core/detect.py b/iambic/core/detect.py new file mode 100644 index 000000000..92a268dcf --- /dev/null +++ b/iambic/core/detect.py @@ -0,0 +1,40 @@ +from collections import defaultdict +from typing import Type + +from iambic.core.models import ProviderChild, BaseTemplate +from iambic.core.utils import evaluate_on_provider + + +def group_detect_messages(group_by: str, messages: list) -> dict: + """Group messages by a key in the message dict. + + Args: + group_by (str): The key to group by. + messages (list): The messages to group. + + Returns: + dict: The grouped messages. + """ + grouped_messages = defaultdict(list) + for message in messages: + grouped_messages[getattr(message, group_by)].append(message) + + return grouped_messages + + +def generate_template_output( + excluded_provider_ids: list[str], + provider_child_map: dict[str, ProviderChild], + template: Type[BaseTemplate] +) -> dict[str, dict]: + provider_children_value_map = dict() + for provider_child_id, provider_child in provider_child_map.items(): + if provider_child_id in excluded_provider_ids: + continue + elif not evaluate_on_provider(template, provider_child, exclude_import_only=False): + continue + + if provider_child_value := template.apply_resource_dict(provider_child): + provider_children_value_map[provider_child_id] = provider_child_value + + return provider_children_value_map diff --git a/iambic/core/models.py b/iambic/core/models.py index dda025097..38e40a931 100644 --- a/iambic/core/models.py +++ b/iambic/core/models.py @@ -29,7 +29,6 @@ import dateparser from deepdiff.model import PrettyOrderedSet from git import Repo -from jinja2 import BaseLoader, Environment from pydantic import BaseModel as PydanticBaseModel from pydantic import Extra, Field, root_validator, schema, validate_model, validator from pydantic.fields import ModelField @@ -41,17 +40,15 @@ apply_to_provider, create_commented_map, get_writable_directory, - sanitize_string, simplify_dt, snake_to_camelcap, sort_dict, transform_comments, - yaml, + yaml, get_rendered_template_str_value, ) if TYPE_CHECKING: from iambic.config.dynamic_config import Config - from iambic.plugins.v0_1_0.aws.models import AWSAccount MappingIntStrAny = typing.Mapping[int | str, Any] AbstractSetIntStr = typing.AbstractSet[int | str] @@ -159,14 +156,14 @@ def get_field_type(field: Any) -> Any: def get_attribute_val_for_account( self, - aws_account: AWSAccount, + provider_child: Type[ProviderChild], attr: str, as_boto_dict: bool = True, ): """ Retrieve the value of an attribute for a specific AWS account. - :param aws_account: The AWSAccount object for which the attribute value should be retrieved. + :param provider_child: The ProviderChild object for which the attribute value should be retrieved. :param attr: The attribute name (supports nested attributes via dot notation, e.g., properties.tags). :param as_boto_dict: If True, the value will be transformed to a boto dictionary if applicable. :return: The attribute value for the specified AWS account. @@ -177,12 +174,12 @@ def get_attribute_val_for_account( attr_val = getattr(attr_val, attr_key) if as_boto_dict and hasattr(attr_val, "_apply_resource_dict"): - return attr_val._apply_resource_dict(aws_account) + return attr_val._apply_resource_dict(provider_child) elif not isinstance(attr_val, list): return attr_val matching_definitions = [ - val for val in attr_val if apply_to_provider(val, aws_account) + val for val in attr_val if apply_to_provider(val, provider_child) ] if len(matching_definitions) == 0: # Fallback to the default definition @@ -194,7 +191,7 @@ def get_attribute_val_for_account( return field.__fields__[split_key[-1]].default elif as_boto_dict: return [ - match._apply_resource_dict(aws_account) + match._apply_resource_dict(provider_child) if hasattr(match, "_apply_resource_dict") else match for match in matching_definitions @@ -202,7 +199,7 @@ def get_attribute_val_for_account( else: return matching_definitions - def _apply_resource_dict(self, aws_account: AWSAccount = None) -> dict: + def _apply_resource_dict(self, provider_child: Type[ProviderChild] = None) -> dict: exclude_keys = { "deleted", "expires_at", @@ -220,10 +217,10 @@ def _apply_resource_dict(self, aws_account: AWSAccount = None) -> dict: exclude_keys.update(self.exclude_keys) has_properties = hasattr(self, "properties") properties = getattr(self, "properties", self) - if aws_account: + if provider_child: resource_dict = { k: self.get_attribute_val_for_account( - aws_account, + provider_child, f"properties.{k}" if has_properties else k, ) for k in properties.__dict__.keys() @@ -239,20 +236,9 @@ def _apply_resource_dict(self, aws_account: AWSAccount = None) -> dict: return {self.case_convention(k): v for k, v in resource_dict.items()} - def apply_resource_dict(self, aws_account: AWSAccount) -> dict: - response = self._apply_resource_dict(aws_account) - variables = {var.key: var.value for var in aws_account.variables} - variables["account_id"] = aws_account.account_id - variables["account_name"] = aws_account.account_name - if hasattr(self, "owner") and (owner := getattr(self, "owner", None)): - variables["owner"] = owner - - rtemplate = Environment(loader=BaseLoader()).from_string(json.dumps(response)) - valid_characters_re = r"[\w_+=,.@-]" - variables = { - k: sanitize_string(v, valid_characters_re) for k, v in variables.items() - } - data = rtemplate.render(var=variables) + def apply_resource_dict(self, provider_child: Type[ProviderChild]) -> dict: + response = self._apply_resource_dict(provider_child) + data = get_rendered_template_str_value(json.dumps(response), provider_child) return json.loads(data) async def remove_expired_resources(self): diff --git a/iambic/core/utils.py b/iambic/core/utils.py index f8b8e88bd..7e1cd7794 100644 --- a/iambic/core/utils.py +++ b/iambic/core/utils.py @@ -17,6 +17,8 @@ import aiofiles import jwt from asgiref.sync import sync_to_async +from jinja2 import BaseLoader +from jinja2.sandbox import ImmutableSandboxedEnvironment from ruamel.yaml import YAML, scalarstring from iambic.core import noq_json as json @@ -26,7 +28,7 @@ from iambic.core.logger import log if TYPE_CHECKING: - from iambic.core.models import ProposedChange + from iambic.core.models import ProposedChange, ProviderChild NOQ_TEMPLATE_REGEX = r".*template_type:\n?.*NOQ::" @@ -892,3 +894,27 @@ def decode_with_reference_time(encoded_jwt, public_key, algorithms, reference_ti ) return payload + + +def get_rendered_template_str_value( + template_value: str, provider_child: typing.Type[ProviderChild] +) -> str: + """ + Render a template string with the variables from the provider child. + """ + valid_characters_re = r"[\w_+=,.@-]" + variables = {var.key: var.value for var in getattr(provider_child, "variables", [])} + for extra_attr in {"account_id", "account_name", "owner"}: + if attr_val := getattr(provider_child, extra_attr, None): + variables[extra_attr] = attr_val + + if not variables: + return template_value + + variables = { + k: sanitize_string(v, valid_characters_re) for k, v in variables.items() + } + rtemplate = ImmutableSandboxedEnvironment(loader=BaseLoader()).from_string( + template_value + ) + return rtemplate.render(var=variables) From cb3db27bf09ca6b12aef5de1ea2e27d08e0fe80c Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Fri, 14 Jul 2023 18:22:43 -0500 Subject: [PATCH 03/11] Fixes to the way roles are resolved as part of iambic detect. --- iambic/core/detect.py | 10 +- iambic/core/models.py | 3 +- .../aws/iam/role/template_generation.py | 93 ++++++++++++------- iambic/plugins/v0_1_0/aws/iam/role/utils.py | 13 ++- 4 files changed, 78 insertions(+), 41 deletions(-) diff --git a/iambic/core/detect.py b/iambic/core/detect.py index 92a268dcf..9b6704c62 100644 --- a/iambic/core/detect.py +++ b/iambic/core/detect.py @@ -1,7 +1,9 @@ +from __future__ import annotations + from collections import defaultdict from typing import Type -from iambic.core.models import ProviderChild, BaseTemplate +from iambic.core.models import BaseTemplate, ProviderChild from iambic.core.utils import evaluate_on_provider @@ -25,13 +27,15 @@ def group_detect_messages(group_by: str, messages: list) -> dict: def generate_template_output( excluded_provider_ids: list[str], provider_child_map: dict[str, ProviderChild], - template: Type[BaseTemplate] + template: Type[BaseTemplate], ) -> dict[str, dict]: provider_children_value_map = dict() for provider_child_id, provider_child in provider_child_map.items(): if provider_child_id in excluded_provider_ids: continue - elif not evaluate_on_provider(template, provider_child, exclude_import_only=False): + elif not evaluate_on_provider( + template, provider_child, exclude_import_only=False + ): continue if provider_child_value := template.apply_resource_dict(provider_child): diff --git a/iambic/core/models.py b/iambic/core/models.py index 38e40a931..5a4c7a383 100644 --- a/iambic/core/models.py +++ b/iambic/core/models.py @@ -39,12 +39,13 @@ LiteralScalarString, apply_to_provider, create_commented_map, + get_rendered_template_str_value, get_writable_directory, simplify_dt, snake_to_camelcap, sort_dict, transform_comments, - yaml, get_rendered_template_str_value, + yaml, ) if TYPE_CHECKING: diff --git a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py index e177508a0..3e9e5b6c9 100644 --- a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import itertools import os from collections import defaultdict @@ -8,13 +9,19 @@ import aiofiles from iambic.core import noq_json as json +from iambic.core.detect import generate_template_output, group_detect_messages from iambic.core.logger import log from iambic.core.models import ExecutionMessage from iambic.core.template_generation import ( create_or_update_template, delete_orphaned_templates, ) -from iambic.core.utils import NoqSemaphore, normalize_dict_keys, resource_file_upsert +from iambic.core.utils import ( + NoqSemaphore, + get_rendered_template_str_value, + normalize_dict_keys, + resource_file_upsert, +) from iambic.plugins.v0_1_0.aws.event_bridge.models import RoleMessageDetails from iambic.plugins.v0_1_0.aws.iam.policy.models import AssumeRolePolicyDocument from iambic.plugins.v0_1_0.aws.iam.role.models import ( @@ -23,7 +30,7 @@ RoleProperties, ) from iambic.plugins.v0_1_0.aws.iam.role.utils import ( - get_role_across_accounts, + get_role, get_role_inline_policies, get_role_managed_policies, list_role_tags, @@ -143,20 +150,39 @@ async def generate_account_role_resource_files( async def generate_role_resource_file_for_all_accounts( exe_message: ExecutionMessage, - aws_accounts: list[AWSAccount], role_name: str, + aws_account_map: dict[str, AWSAccount], + updated_account_ids: list[str], + iambic_template: AwsIamRoleTemplate, ) -> list: + async def get_role_for_account(aws_account: AWSAccount): + iam_client = await aws_account.get_boto3_client("iam") + account_role_name = get_rendered_template_str_value(role_name, aws_account) + role = await get_role(account_role_name, iam_client) + role["PermissionsBoundary"]["PolicyArn"] = role["PermissionsBoundary"].pop( + "PermissionsBoundaryArn" + ) + return {aws_account.account_id: role} + account_resource_dir_map = { aws_account.account_id: get_response_dir(exe_message, aws_account) - for aws_account in aws_accounts + for aws_account in aws_account_map.values() } role_resource_file_upsert_semaphore = NoqSemaphore(resource_file_upsert, 10) messages = [] response = [] - role_across_accounts = await get_role_across_accounts( - aws_accounts, role_name, False + role_across_accounts = generate_template_output( + updated_account_ids, aws_account_map, iambic_template ) + updated_account_roles = await asyncio.gather( + *[ + get_role_for_account(aws_account_map[account_id]) + for account_id in updated_account_ids + ] + ) + for updated_account_role in updated_account_roles: + role_across_accounts.update(updated_account_role) role_across_accounts = {k: v for k, v in role_across_accounts.items() if v} log.debug( @@ -450,7 +476,7 @@ async def create_templated_role( # noqa: C901 AwsIamRoleTemplate, role_template_params, RoleProperties(**role_template_properties), - list(aws_account_map.values()), + [aws_account_map[role_ref["account_id"]] for role_ref in role_refs], ) except Exception as e: log_params = { @@ -491,18 +517,21 @@ async def collect_aws_roles( ) if detect_messages: - aws_accounts = list(aws_account_map.values()) generate_role_resource_file_for_all_accounts_semaphore = NoqSemaphore( generate_role_resource_file_for_all_accounts, 45 ) + grouped_detect_messages = group_detect_messages("role_name", detect_messages) tasks = [ { "exe_message": exe_message, - "aws_accounts": aws_accounts, - "role_name": role.role_name, + "aws_account_map": aws_account_map, + "role_name": role_name, + "updated_account_ids": [ + message.account_id for message in role_messages + ], + "iambic_template": existing_template_map.get(role_name), } - for role in detect_messages - if not role.delete + for role_name, role_messages in grouped_detect_messages.items() ] # Remove deleted or mark templates for update @@ -519,15 +548,6 @@ async def collect_aws_roles( ): # It's the only account for the template so delete it existing_template.delete() - else: - # There are other accounts for the template so re-eval the template - tasks.append( - { - "exe_message": exe_message, - "aws_accounts": aws_accounts, - "role_name": existing_template.properties.role_name, - } - ) account_role_list = ( await generate_role_resource_file_for_all_accounts_semaphore.process(tasks) @@ -568,19 +588,24 @@ async def collect_aws_roles( } ) - log.info( - "Setting inline policies in role templates", - accounts=list(aws_account_map.keys()), - ) - await set_role_resource_inline_policies_semaphore.process(messages) - log.info( - "Setting managed policies in role templates", - accounts=list(aws_account_map.keys()), - ) - await set_role_resource_managed_policies_semaphore.process(messages) - log.info("Setting tags in role templates", accounts=list(aws_account_map.keys())) - await set_role_resource_tags_semaphore.process(messages) - log.info("Finished retrieving role details", accounts=list(aws_account_map.keys())) + if not detect_messages: + log.info( + "Setting inline policies in role templates", + accounts=list(aws_account_map.keys()), + ) + await set_role_resource_inline_policies_semaphore.process(messages) + log.info( + "Setting managed policies in role templates", + accounts=list(aws_account_map.keys()), + ) + await set_role_resource_managed_policies_semaphore.process(messages) + log.info( + "Setting tags in role templates", accounts=list(aws_account_map.keys()) + ) + await set_role_resource_tags_semaphore.process(messages) + log.info( + "Finished retrieving role details", accounts=list(aws_account_map.keys()) + ) account_role_output = json.dumps(account_roles) with open( diff --git a/iambic/plugins/v0_1_0/aws/iam/role/utils.py b/iambic/plugins/v0_1_0/aws/iam/role/utils.py index de886da8a..2b233564f 100644 --- a/iambic/plugins/v0_1_0/aws/iam/role/utils.py +++ b/iambic/plugins/v0_1_0/aws/iam/role/utils.py @@ -10,7 +10,11 @@ from iambic.core.context import ctx from iambic.core.logger import log from iambic.core.models import ProposedChange, ProposedChangeType -from iambic.core.utils import aio_wrapper, plugin_apply_wrapper +from iambic.core.utils import ( + aio_wrapper, + get_rendered_template_str_value, + plugin_apply_wrapper, +) from iambic.plugins.v0_1_0.aws.models import AWSAccount from iambic.plugins.v0_1_0.aws.utils import boto_crud_call, paginated_search @@ -99,7 +103,7 @@ async def get_role_managed_policies(role_name: str, iam_client) -> list[dict[str else: break - return policies + return [{"PolicyArn": policy["PolicyArn"]} for policy in policies] async def get_role(role_name: str, iam_client, include_policies: bool = True) -> dict: @@ -107,6 +111,8 @@ async def get_role(role_name: str, iam_client, include_policies: bool = True) -> current_role = (await boto_crud_call(iam_client.get_role, RoleName=role_name))[ "Role" ] + current_role.get("PermissionsBoundary", {}).pop("PermissionsBoundaryType", None) + if include_policies: current_role["ManagedPolicies"] = await get_role_managed_policies( role_name, iam_client @@ -129,9 +135,10 @@ async def get_role_across_accounts( ) -> dict: async def get_role_for_account(aws_account: AWSAccount): iam_client = await aws_account.get_boto3_client("iam") + account_role_name = get_rendered_template_str_value(role_name, aws_account) return { aws_account.account_id: await get_role( - role_name, iam_client, include_policies + account_role_name, iam_client, include_policies ) } From 6fb73a201c98251601dc9a2ffbca487f9afe438b Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:50:29 -0500 Subject: [PATCH 04/11] Make template optional in generate_template_output. Performing a recursive sort on the provider child dicts in base_group_dict_attribute. --- iambic/core/detect.py | 6 ++++-- iambic/core/template_generation.py | 20 ++++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/iambic/core/detect.py b/iambic/core/detect.py index 9b6704c62..ab826b222 100644 --- a/iambic/core/detect.py +++ b/iambic/core/detect.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import defaultdict -from typing import Type +from typing import Optional, Type from iambic.core.models import BaseTemplate, ProviderChild from iambic.core.utils import evaluate_on_provider @@ -27,9 +27,11 @@ def group_detect_messages(group_by: str, messages: list) -> dict: def generate_template_output( excluded_provider_ids: list[str], provider_child_map: dict[str, ProviderChild], - template: Type[BaseTemplate], + template: Optional[Type[BaseTemplate]], ) -> dict[str, dict]: provider_children_value_map = dict() + if not template: + return provider_children_value_map for provider_child_id, provider_child in provider_child_map.items(): if provider_child_id in excluded_provider_ids: continue diff --git a/iambic/core/template_generation.py b/iambic/core/template_generation.py index 817b281e6..fb1878d74 100644 --- a/iambic/core/template_generation.py +++ b/iambic/core/template_generation.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections import defaultdict +from collections import OrderedDict, defaultdict from typing import Type, Union import xxhash @@ -20,6 +20,22 @@ ) +def deep_sort(obj: Union[dict, list]) -> Union[dict, list]: + """Recursively sorts a dict or list""" + if isinstance(obj, dict): + obj = OrderedDict(sorted(obj.items())) + for k, v in obj.items(): + if isinstance(v, dict) or isinstance(v, list): + obj[k] = deep_sort(v) + elif isinstance(obj, list): + for i, v in enumerate(obj): + if isinstance(v, dict) or isinstance(v, list): + obj[i] = deep_sort(v) + obj = sorted(obj, key=lambda x: json.dumps(x)) + + return obj + + async def get_existing_template_map( repo_dir: str, template_type: str, @@ -275,7 +291,7 @@ async def base_group_dict_attribute( ] = provider_child_resource[provider_child_key_id] # Set raw dict resource_hash = xxhash.xxh32( - json.dumps(resource["resource_val"]) + json.dumps(deep_sort(resource["resource_val"])) ).hexdigest() hash_map[resource_hash] = resource["resource_val"] # Set dict with attempted interpolation From cc542b701df518e40c2caa56db16ec4c8beef52f Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:52:52 -0500 Subject: [PATCH 05/11] Fixes to the IAM Groups detect behavior. --- .../aws/iam/group/template_generation.py | 87 ++++++++++++------- iambic/plugins/v0_1_0/aws/iam/group/utils.py | 11 ++- .../aws/iam/group/test_template_generation.py | 6 +- .../iam/policy/test_template_generation.py | 13 ++- .../aws/iam/role/test_template_generation.py | 6 +- .../aws/iam/user/test_template_generation.py | 6 +- 6 files changed, 90 insertions(+), 39 deletions(-) diff --git a/iambic/plugins/v0_1_0/aws/iam/group/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/group/template_generation.py index 4f49c9679..9ae109be1 100644 --- a/iambic/plugins/v0_1_0/aws/iam/group/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/group/template_generation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import itertools import os from collections import defaultdict @@ -8,13 +9,19 @@ import aiofiles from iambic.core import noq_json as json +from iambic.core.detect import generate_template_output, group_detect_messages from iambic.core.logger import log from iambic.core.models import ExecutionMessage from iambic.core.template_generation import ( create_or_update_template, delete_orphaned_templates, ) -from iambic.core.utils import NoqSemaphore, normalize_dict_keys, resource_file_upsert +from iambic.core.utils import ( + NoqSemaphore, + get_rendered_template_str_value, + normalize_dict_keys, + resource_file_upsert, +) from iambic.plugins.v0_1_0.aws.event_bridge.models import GroupMessageDetails from iambic.plugins.v0_1_0.aws.iam.group.models import ( AWS_IAM_GROUP_TEMPLATE_TYPE, @@ -22,7 +29,7 @@ GroupProperties, ) from iambic.plugins.v0_1_0.aws.iam.group.utils import ( - get_group_across_accounts, + get_group, get_group_inline_policies, get_group_managed_policies, list_groups, @@ -140,19 +147,36 @@ async def generate_account_group_resource_files( async def generate_group_resource_file_for_all_accounts( - exe_message: ExecutionMessage, aws_accounts: list[AWSAccount], group_name: str + exe_message: ExecutionMessage, + group_name: str, + aws_account_map: dict[str, AWSAccount], + updated_account_ids: list[str], + iambic_template: Optional[AwsIamGroupTemplate], ) -> list: + async def get_group_for_account(aws_account: AWSAccount): + iam_client = await aws_account.get_boto3_client("iam") + account_group_name = get_rendered_template_str_value(group_name, aws_account) + return {aws_account.account_id: await get_group(account_group_name, iam_client)} + account_group_response_dir_map = { aws_account.account_id: get_response_dir(exe_message, aws_account) - for aws_account in aws_accounts + for aws_account in aws_account_map.values() } group_resource_file_upsert_semaphore = NoqSemaphore(resource_file_upsert, 10) messages = [] response = [] - group_across_accounts = await get_group_across_accounts( - aws_accounts, group_name, False + group_across_accounts = generate_template_output( + updated_account_ids, aws_account_map, iambic_template + ) + updated_account_groups = await asyncio.gather( + *[ + get_group_for_account(aws_account_map[account_id]) + for account_id in updated_account_ids + ] ) + for updated_account_group in updated_account_groups: + group_across_accounts.update(updated_account_group) group_across_accounts = {k: v for k, v in group_across_accounts.items() if v} log.debug( @@ -344,7 +368,7 @@ async def create_templated_group( # noqa: C901 AwsIamGroupTemplate, group_template_params, GroupProperties(**group_template_properties), - list(aws_account_map.values()), + [aws_account_map[group_ref["account_id"]] for group_ref in group_refs], ) @@ -381,14 +405,21 @@ async def collect_aws_groups( ) if detect_messages: - aws_accounts = list(aws_account_map.values()) generate_group_resource_file_for_all_accounts_semaphore = NoqSemaphore( generate_group_resource_file_for_all_accounts, 45 ) + grouped_detect_messages = group_detect_messages("group_name", detect_messages) tasks = [ - {"aws_accounts": aws_accounts, "group_name": group.group_name} - for group in detect_messages - if not group.delete + { + "exe_message": exe_message, + "aws_account_map": aws_account_map, + "group_name": group_name, + "updated_account_ids": [ + message.account_id for message in group_messages + ], + "iambic_template": existing_template_map.get(group_name), + } + for group_name, group_messages in grouped_detect_messages.items() ] # Remove deleted or mark templates for update @@ -405,15 +436,6 @@ async def collect_aws_groups( ): # It's the only account for the template so delete it existing_template.delete() - else: - # There are other accounts for the template so re-eval the template - tasks.append( - { - "exe_message": exe_message, - "aws_accounts": aws_accounts, - "group_name": existing_template.properties.group_name, - } - ) account_group_list = ( await generate_group_resource_file_for_all_accounts_semaphore.process(tasks) @@ -461,17 +483,20 @@ async def collect_aws_groups( } ) - log.info( - "Setting inline policies in group templates", - accounts=list(aws_account_map.keys()), - ) - await set_group_resource_inline_policies_semaphore.process(messages) - log.info( - "Setting managed policies in group templates", - accounts=list(aws_account_map.keys()), - ) - await set_group_resource_managed_policies_semaphore.process(messages) - log.info("Finished retrieving group details", accounts=list(aws_account_map.keys())) + if not detect_messages: + log.info( + "Setting inline policies in group templates", + accounts=list(aws_account_map.keys()), + ) + await set_group_resource_inline_policies_semaphore.process(messages) + log.info( + "Setting managed policies in group templates", + accounts=list(aws_account_map.keys()), + ) + await set_group_resource_managed_policies_semaphore.process(messages) + log.info( + "Finished retrieving group details", accounts=list(aws_account_map.keys()) + ) account_group_output = json.dumps(account_groups) with open( diff --git a/iambic/plugins/v0_1_0/aws/iam/group/utils.py b/iambic/plugins/v0_1_0/aws/iam/group/utils.py index 1d3ba1769..ba1f30865 100644 --- a/iambic/plugins/v0_1_0/aws/iam/group/utils.py +++ b/iambic/plugins/v0_1_0/aws/iam/group/utils.py @@ -10,7 +10,11 @@ from iambic.core.context import ctx from iambic.core.logger import log from iambic.core.models import ProposedChange, ProposedChangeType -from iambic.core.utils import aio_wrapper, plugin_apply_wrapper +from iambic.core.utils import ( + aio_wrapper, + get_rendered_template_str_value, + plugin_apply_wrapper, +) from iambic.plugins.v0_1_0.aws.utils import boto_crud_call, paginated_search if TYPE_CHECKING: @@ -73,7 +77,7 @@ async def get_group_managed_policies( else: break - return policies + return [{"PolicyArn": policy["PolicyArn"]} for policy in policies] async def get_group(group_name: str, iam_client, include_policies: bool = True) -> dict: @@ -99,9 +103,10 @@ async def get_group_across_accounts( ) -> dict: async def get_group_for_account(aws_account: AWSAccount): iam_client = await aws_account.get_boto3_client("iam") + account_group_name = get_rendered_template_str_value(group_name, aws_account) return { aws_account.account_id: await get_group( - group_name, iam_client, include_policies + account_group_name, iam_client, include_policies ) } diff --git a/test/plugins/v0_1_0/aws/iam/group/test_template_generation.py b/test/plugins/v0_1_0/aws/iam/group/test_template_generation.py index 58f443c44..61879ceab 100644 --- a/test/plugins/v0_1_0/aws/iam/group/test_template_generation.py +++ b/test/plugins/v0_1_0/aws/iam/group/test_template_generation.py @@ -112,7 +112,11 @@ async def test_generate_group_resource_file_for_all_accounts( ): _, templates_base_dir = mock_fs files = await generate_group_resource_file_for_all_accounts( - mock_execution_message, [mock_aws_account], EXAMPLE_GROUPNAME + mock_execution_message, + EXAMPLE_GROUPNAME, + {mock_aws_account.account_id: mock_aws_account}, + [mock_aws_account.account_id], + None, ) assert len(files) == 1 assert files[0]["name"] == EXAMPLE_GROUPNAME diff --git a/test/plugins/v0_1_0/aws/iam/policy/test_template_generation.py b/test/plugins/v0_1_0/aws/iam/policy/test_template_generation.py index a29b0d304..d319e0a1f 100644 --- a/test/plugins/v0_1_0/aws/iam/policy/test_template_generation.py +++ b/test/plugins/v0_1_0/aws/iam/policy/test_template_generation.py @@ -17,6 +17,7 @@ from iambic.core.iambic_enum import Command from iambic.core.models import ExecutionMessage from iambic.core.template_generation import get_existing_template_map +from iambic.plugins.v0_1_0.aws.event_bridge.models import ManagedPolicyMessageDetails from iambic.plugins.v0_1_0.aws.iam.policy.template_generation import ( collect_aws_managed_policies, generate_account_managed_policy_resource_files, @@ -115,9 +116,17 @@ async def test_generate_managed_policy_resource_file_for_all_accounts( policy_path = "/" files = await generate_managed_policy_resource_file_for_all_accounts( mock_execution_message, - [mock_aws_account], - policy_path, EXAMPLE_MANAGED_POLICY_NAME, + {mock_aws_account.account_id: mock_aws_account}, + [ + ManagedPolicyMessageDetails( + policy_path=policy_path, + account_id=mock_aws_account.account_id, + policy_name=EXAMPLE_MANAGED_POLICY_NAME, + delete=False, + ) + ], + None, ) assert len(files) == 1 assert files[0]["policy_name"] == EXAMPLE_MANAGED_POLICY_NAME diff --git a/test/plugins/v0_1_0/aws/iam/role/test_template_generation.py b/test/plugins/v0_1_0/aws/iam/role/test_template_generation.py index a03bea315..66d3937f7 100644 --- a/test/plugins/v0_1_0/aws/iam/role/test_template_generation.py +++ b/test/plugins/v0_1_0/aws/iam/role/test_template_generation.py @@ -191,7 +191,11 @@ async def test_generate_role_resource_file_for_all_accounts( ): _, templates_base_dir = mock_fs files = await generate_role_resource_file_for_all_accounts( - mock_execution_message, [mock_aws_account], EXAMPLE_ROLE_NAME + mock_execution_message, + EXAMPLE_ROLE_NAME, + {mock_aws_account.account_id: mock_aws_account}, + [mock_aws_account.account_id], + None, ) assert len(files) == 1 assert files[0]["name"] == EXAMPLE_ROLE_NAME diff --git a/test/plugins/v0_1_0/aws/iam/user/test_template_generation.py b/test/plugins/v0_1_0/aws/iam/user/test_template_generation.py index 968960148..59be41968 100644 --- a/test/plugins/v0_1_0/aws/iam/user/test_template_generation.py +++ b/test/plugins/v0_1_0/aws/iam/user/test_template_generation.py @@ -115,7 +115,11 @@ async def test_generate_user_resource_file_for_all_accounts( ): _, templates_base_dir = mock_fs files = await generate_user_resource_file_for_all_accounts( - mock_execution_message, [mock_aws_account], EXAMPLE_USERNAME + mock_execution_message, + EXAMPLE_USERNAME, + {mock_aws_account.account_id: mock_aws_account}, + [mock_aws_account.account_id], + None, ) assert len(files) == 1 assert files[0]["name"] == EXAMPLE_USERNAME From c2212775883702e72f8b529ce9de26abf44910aa Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:54:32 -0500 Subject: [PATCH 06/11] Fixes to the IAM Managed Policy detect behavior. --- .../plugins/v0_1_0/aws/iam/policy/models.py | 6 ++ .../aws/iam/policy/template_generation.py | 85 ++++++++++++------- 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/iambic/plugins/v0_1_0/aws/iam/policy/models.py b/iambic/plugins/v0_1_0/aws/iam/policy/models.py index 1d135f52e..484eccb10 100644 --- a/iambic/plugins/v0_1_0/aws/iam/policy/models.py +++ b/iambic/plugins/v0_1_0/aws/iam/policy/models.py @@ -318,6 +318,12 @@ def get_arn_for_account(self, aws_account: AWSAccount) -> str: def _apply_resource_dict(self, aws_account: AWSAccount = None) -> dict: resource_dict = super()._apply_resource_dict(aws_account) resource_dict["Arn"] = self.get_arn_for_account(aws_account) + + if policy_document := resource_dict.pop("PolicyDocument", []): + if isinstance(policy_document, list): + policy_document = policy_document[0] + resource_dict["PolicyDocument"] = policy_document + return resource_dict async def _apply_to_account(self, aws_account: AWSAccount) -> AccountChangeDetails: diff --git a/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py index 107b447b1..edb83f8a0 100644 --- a/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import itertools import os from collections import defaultdict @@ -8,13 +9,19 @@ import aiofiles from iambic.core import noq_json as json +from iambic.core.detect import generate_template_output, group_detect_messages from iambic.core.logger import log from iambic.core.models import ExecutionMessage from iambic.core.template_generation import ( create_or_update_template, delete_orphaned_templates, ) -from iambic.core.utils import NoqSemaphore, normalize_dict_keys, resource_file_upsert +from iambic.core.utils import ( + NoqSemaphore, + get_rendered_template_str_value, + normalize_dict_keys, + resource_file_upsert, +) from iambic.plugins.v0_1_0.aws.event_bridge.models import ManagedPolicyMessageDetails from iambic.plugins.v0_1_0.aws.iam.policy.models import ( AWS_MANAGED_POLICY_TEMPLATE_TYPE, @@ -22,7 +29,7 @@ ManagedPolicyProperties, ) from iambic.plugins.v0_1_0.aws.iam.policy.utils import ( - get_managed_policy_across_accounts, + get_managed_policy, list_managed_policies, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount @@ -142,39 +149,68 @@ async def generate_account_managed_policy_resource_files( async def generate_managed_policy_resource_file_for_all_accounts( exe_message: ExecutionMessage, - aws_accounts: list[AWSAccount], - policy_path: str, policy_name: str, + aws_account_map: dict[str, AWSAccount], + policy_messages: list[ManagedPolicyMessageDetails], + iambic_template: Optional[AwsIamManagedPolicyTemplate], ) -> list: + async def get_managed_policy_for_account( + aws_account: AWSAccount, managed_policy_path: str + ): + iam_client = await aws_account.get_boto3_client("iam") + arn = f"arn:aws:iam::{aws_account.account_id}:policy{managed_policy_path}{policy_name}" + account_arn = get_rendered_template_str_value(arn, aws_account) + return { + aws_account.account_id: await get_managed_policy(iam_client, account_arn) + } + account_resource_dir_map = { aws_account.account_id: get_response_dir(exe_message, aws_account) - for aws_account in aws_accounts + for aws_account in aws_account_map.values() } mp_resource_file_upsert_semaphore = NoqSemaphore(resource_file_upsert, 10) messages = [] response = [] - mp_across_accounts = await get_managed_policy_across_accounts( - aws_accounts, policy_path, policy_name + mp_across_accounts = generate_template_output( + [msg.account_id for msg in policy_messages], aws_account_map, iambic_template ) + for account_id, account_mp in mp_across_accounts.items(): + print(account_mp["PolicyDocument"]) + mp_across_accounts[account_id]["PolicyDocument"] = json.loads( + account_mp["PolicyDocument"] + ) + + updated_account_mps = await asyncio.gather( + *[ + get_managed_policy_for_account( + aws_account_map[msg.account_id], msg.policy_path + ) + for msg in policy_messages + ] + ) + for updated_account_mp in updated_account_mps: + mp_across_accounts.update(updated_account_mp) mp_across_accounts = {k: v for k, v in mp_across_accounts.items() if v} + with open("mp_across_accounts.json", "w") as f: + f.write(json.dumps(mp_across_accounts, indent=2)) + log.debug( "Retrieved AWS IAM Managed Policy for all accounts.", policy_name=policy_name, - policy_path=policy_path, total_accounts=len(mp_across_accounts), ) for account_id, managed_policy in mp_across_accounts.items(): - policy_path = os.path.join( + policy_file_path = os.path.join( account_resource_dir_map[account_id], f'{managed_policy["PolicyName"]}.json', ) response.append( { - "file_path": policy_path, + "file_path": policy_file_path, "policy_name": managed_policy["PolicyName"], "arn": managed_policy["Arn"], "account_id": account_id, @@ -182,7 +218,9 @@ async def generate_managed_policy_resource_file_for_all_accounts( ) messages.append( dict( - file_path=policy_path, content_as_dict=managed_policy, replace_file=True + file_path=policy_file_path, + content_as_dict=managed_policy, + replace_file=True, ) ) @@ -190,7 +228,6 @@ async def generate_managed_policy_resource_file_for_all_accounts( log.debug( "Finished caching AWS IAM Managed Policy for all accounts.", policy_name=policy_name, - policy_path=policy_path, total_accounts=len(mp_across_accounts), ) @@ -330,7 +367,7 @@ async def create_templated_managed_policy( # noqa: C901 AwsIamManagedPolicyTemplate, template_params, ManagedPolicyProperties(**template_properties), - list(aws_account_map.values()), + [aws_account_map[mp_ref["account_id"]] for mp_ref in managed_policy_refs], ) @@ -363,20 +400,20 @@ async def collect_aws_managed_policies( ) if detect_messages: - aws_accounts = list(aws_account_map.values()) generate_mp_resource_file_for_all_accounts_semaphore = NoqSemaphore( generate_managed_policy_resource_file_for_all_accounts, 50 ) + grouped_detect_messages = group_detect_messages("policy_name", detect_messages) tasks = [ { "exe_message": exe_message, - "aws_accounts": aws_accounts, - "policy_path": managed_policy.policy_path, - "policy_name": managed_policy.policy_name, + "aws_account_map": aws_account_map, + "policy_name": policy_name, + "policy_messages": policy_messages, + "iambic_template": existing_template_map.get(policy_name), } - for managed_policy in detect_messages - if not managed_policy.delete + for policy_name, policy_messages in grouped_detect_messages.items() ] # Remove deleted or mark templates for update @@ -399,16 +436,6 @@ async def collect_aws_managed_policies( ): # It's the only account for the template so delete it existing_template.delete() - else: - # There are other accounts for the template so re-eval the template - tasks.append( - { - "exe_message": exe_message, - "aws_accounts": aws_accounts, - "policy_path": existing_template.properties.path, - "policy_name": existing_template.properties.policy_name, - } - ) account_mp_list = ( await generate_mp_resource_file_for_all_accounts_semaphore.process(tasks) From def328c56d44089bad9a2ebb004175d662bc98bd Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:58:07 -0500 Subject: [PATCH 07/11] Fixes to the IAM Users detect behavior. Clean up generate_role_resource_file_for_all_accounts. --- .../aws/iam/role/template_generation.py | 9 +- .../aws/iam/user/template_generation.py | 89 +++++++++++-------- iambic/plugins/v0_1_0/aws/iam/user/utils.py | 2 +- 3 files changed, 60 insertions(+), 40 deletions(-) diff --git a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py index 3e9e5b6c9..08815b3a7 100644 --- a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py @@ -153,15 +153,16 @@ async def generate_role_resource_file_for_all_accounts( role_name: str, aws_account_map: dict[str, AWSAccount], updated_account_ids: list[str], - iambic_template: AwsIamRoleTemplate, + iambic_template: Optional[AwsIamRoleTemplate], ) -> list: async def get_role_for_account(aws_account: AWSAccount): iam_client = await aws_account.get_boto3_client("iam") account_role_name = get_rendered_template_str_value(role_name, aws_account) role = await get_role(account_role_name, iam_client) - role["PermissionsBoundary"]["PolicyArn"] = role["PermissionsBoundary"].pop( - "PermissionsBoundaryArn" - ) + if "PermissionsBoundary" in role: + role["PermissionsBoundary"]["PolicyArn"] = role["PermissionsBoundary"].pop( + "PermissionsBoundaryArn" + ) return {aws_account.account_id: role} account_resource_dir_map = { diff --git a/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py index d950b48a7..72d6c6e92 100644 --- a/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import itertools import os from collections import defaultdict @@ -8,6 +9,7 @@ import aiofiles from iambic.core import noq_json as json +from iambic.core.detect import generate_template_output, group_detect_messages from iambic.core.logger import log from iambic.core.models import ExecutionMessage from iambic.core.template_generation import ( @@ -22,7 +24,7 @@ UserProperties, ) from iambic.plugins.v0_1_0.aws.iam.user.utils import ( - get_user_across_accounts, + get_user, get_user_groups, get_user_inline_policies, get_user_managed_policies, @@ -142,19 +144,35 @@ async def generate_account_user_resource_files( async def generate_user_resource_file_for_all_accounts( - exe_message: ExecutionMessage, aws_accounts: list[AWSAccount], user_name: str + exe_message: ExecutionMessage, + user_name: str, + aws_account_map: dict[str, AWSAccount], + updated_account_ids: list[str], + iambic_template: Optional[AwsIamUserTemplate], ) -> list: + async def get_user_for_account(aws_account: AWSAccount): + iam_client = await aws_account.get_boto3_client("iam") + return {aws_account.account_id: await get_user(user_name, iam_client)} + account_resource_dir_map = { aws_account.account_id: get_response_dir(exe_message, aws_account) - for aws_account in aws_accounts + for aws_account in aws_account_map.values() } user_resource_file_upsert_semaphore = NoqSemaphore(resource_file_upsert, 10) messages = [] response = [] - user_across_accounts = await get_user_across_accounts( - aws_accounts, user_name, False + user_across_accounts = generate_template_output( + updated_account_ids, aws_account_map, iambic_template ) + updated_account_users = await asyncio.gather( + *[ + get_user_for_account(aws_account_map[account_id]) + for account_id in updated_account_ids + ] + ) + for updated_account_user in updated_account_users: + user_across_accounts.update(updated_account_user) user_across_accounts = {k: v for k, v in user_across_accounts.items() if v} log.debug( @@ -427,7 +445,7 @@ async def create_templated_user( # noqa: C901 AwsIamUserTemplate, user_template_params, UserProperties(**user_template_properties), - list(aws_account_map.values()), + [aws_account_map[user_ref["account_id"]] for user_ref in user_refs], ) @@ -460,18 +478,21 @@ async def collect_aws_users( ) if detect_messages: - aws_accounts = list(aws_account_map.values()) generate_user_resource_file_for_all_accounts_semaphore = NoqSemaphore( generate_user_resource_file_for_all_accounts, 50 ) + grouped_detect_messages = group_detect_messages("user_name", detect_messages) tasks = [ { "exe_message": exe_message, - "aws_accounts": aws_accounts, - "user_name": user.user_name, + "aws_account_map": aws_account_map, + "user_name": user_name, + "updated_account_ids": [ + message.account_id for message in user_messages + ], + "iambic_template": existing_template_map.get(user_name), } - for user in detect_messages - if not user.delete + for user_name, user_messages in grouped_detect_messages.items() ] # Remove deleted or mark templates for update @@ -488,15 +509,6 @@ async def collect_aws_users( ): # It's the only account for the template so delete it existing_template.delete() - else: - # There are other accounts for the template so re-eval the template - tasks.append( - { - "exe_message": exe_message, - "aws_accounts": aws_accounts, - "user_name": existing_template.properties.user_name, - } - ) account_user_list = ( await generate_user_resource_file_for_all_accounts_semaphore.process(tasks) @@ -537,21 +549,28 @@ async def collect_aws_users( } ) - log.info( - "Setting inline policies in user templates", - accounts=list(aws_account_map.keys()), - ) - await set_user_resource_inline_policies_semaphore.process(messages) - log.info( - "Setting managed policies in user templates", - accounts=list(aws_account_map.keys()), - ) - await set_user_resource_managed_policies_semaphore.process(messages) - log.info("Setting groups in user templates", accounts=list(aws_account_map.keys())) - await set_user_resource_groups_semaphore.process(messages) - log.info("Setting tags in user templates", accounts=list(aws_account_map.keys())) - await set_user_resource_tags_semaphore.process(messages) - log.info("Finished retrieving user details", accounts=list(aws_account_map.keys())) + if not detect_messages: + log.info( + "Setting inline policies in user templates", + accounts=list(aws_account_map.keys()), + ) + await set_user_resource_inline_policies_semaphore.process(messages) + log.info( + "Setting managed policies in user templates", + accounts=list(aws_account_map.keys()), + ) + await set_user_resource_managed_policies_semaphore.process(messages) + log.info( + "Setting groups in user templates", accounts=list(aws_account_map.keys()) + ) + await set_user_resource_groups_semaphore.process(messages) + log.info( + "Setting tags in user templates", accounts=list(aws_account_map.keys()) + ) + await set_user_resource_tags_semaphore.process(messages) + log.info( + "Finished retrieving user details", accounts=list(aws_account_map.keys()) + ) account_user_output = json.dumps(account_users) with open( diff --git a/iambic/plugins/v0_1_0/aws/iam/user/utils.py b/iambic/plugins/v0_1_0/aws/iam/user/utils.py index d6b9895f8..f538021b0 100644 --- a/iambic/plugins/v0_1_0/aws/iam/user/utils.py +++ b/iambic/plugins/v0_1_0/aws/iam/user/utils.py @@ -85,7 +85,7 @@ async def get_user_managed_policies(user_name: str, iam_client) -> list[dict[str else: break - return policies + return [{"PolicyArn": policy["PolicyArn"]} for policy in policies] async def get_user(user_name: str, iam_client, include_policies: bool = True) -> dict: From 6c7e2f97b000e532fea2000d3fd9b78bb030bf89 Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 17 Jul 2023 10:58:39 -0500 Subject: [PATCH 08/11] Fixes to the IdentityCenter PermissionSet import behavior. --- .../aws/identity_center/permission_set/template_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py index c28bc86d5..67b264b62 100644 --- a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py @@ -387,7 +387,7 @@ async def create_templated_permission_set( # noqa: C901 AwsIdentityCenterPermissionSetTemplate, template_params, PermissionSetProperties(**template_properties), - list(aws_account_map.values()), + [aws_account_map[ps_ref["account_id"]] for ps_ref in permission_set_refs], ) From 83716d0dca7da9ce3197340f47925b3d7a161266 Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 17 Jul 2023 11:16:09 -0500 Subject: [PATCH 09/11] Updated CHANGELOG --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a80c06b73..b9b6399eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Change Log +## 0.10.11 (July 17, 2023) + +BUG FIXES: +* Fixed race condition on iambic detect not using templated resource id grouping resources +* Fixed issue where a resource could show as excluded on a resource it was never evaluated on + +ENHANCEMENTS: +* Improved ordering of template attributes +* `base_group_dict_attribute` is now more deterministic in its grouping +* `iambic detect` performance optimizations. + * Now only evaluates on the account a resource id change is detected on as opposed to all accounts. + * Example if `engineering` is on all accounts and detect is ran for account a, only `engineering` on account a is evaluated. + ## 0.10.1 (July 3, 2023) DOCS: From a37cd337cdbabe3b9527564903272c52abe10da3 Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Mon, 17 Jul 2023 11:32:26 -0500 Subject: [PATCH 10/11] Updated CHANGELOG --- CHANGELOG.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9b6399eb..27a629b68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,15 +4,16 @@ ## 0.10.11 (July 17, 2023) BUG FIXES: -* Fixed race condition on iambic detect not using templated resource id grouping resources -* Fixed issue where a resource could show as excluded on a resource it was never evaluated on +* Fixed race condition on iambic detect not using templated resource id grouping resources. +* Fixed issue where a resource could show as excluded on a resource it was never evaluated on. ENHANCEMENTS: -* Improved ordering of template attributes -* `base_group_dict_attribute` is now more deterministic in its grouping -* `iambic detect` performance optimizations. +* Improved ordering of template attributes. +* `base_group_dict_attribute` is now more deterministic in its grouping. +* `iambic detect` performance optimizations. * Now only evaluates on the account a resource id change is detected on as opposed to all accounts. * Example if `engineering` is on all accounts and detect is ran for account a, only `engineering` on account a is evaluated. +* Removed remaining AWS provider references from core. ## 0.10.1 (July 3, 2023) From 78e1ec3c1f4da17ac2df5969bcf13e0e9bf50912 Mon Sep 17 00:00:00 2001 From: Will-NOQ <102608210+Will-NOQ@users.noreply.github.com> Date: Tue, 18 Jul 2023 08:39:12 -0500 Subject: [PATCH 11/11] [aws_account_map[mp_ref["account_id"]] for mp_ref in managed_policy_refs] -> list(aws_account_map.values()) --- iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py | 2 +- iambic/plugins/v0_1_0/aws/iam/role/template_generation.py | 2 +- iambic/plugins/v0_1_0/aws/iam/user/template_generation.py | 2 +- .../aws/identity_center/permission_set/template_generation.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py index edb83f8a0..d9374a810 100644 --- a/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/policy/template_generation.py @@ -367,7 +367,7 @@ async def create_templated_managed_policy( # noqa: C901 AwsIamManagedPolicyTemplate, template_params, ManagedPolicyProperties(**template_properties), - [aws_account_map[mp_ref["account_id"]] for mp_ref in managed_policy_refs], + list(aws_account_map.values()), ) diff --git a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py index 08815b3a7..4a733431b 100644 --- a/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/role/template_generation.py @@ -477,7 +477,7 @@ async def create_templated_role( # noqa: C901 AwsIamRoleTemplate, role_template_params, RoleProperties(**role_template_properties), - [aws_account_map[role_ref["account_id"]] for role_ref in role_refs], + list(aws_account_map.values()), ) except Exception as e: log_params = { diff --git a/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py b/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py index 72d6c6e92..d51434e76 100644 --- a/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/iam/user/template_generation.py @@ -445,7 +445,7 @@ async def create_templated_user( # noqa: C901 AwsIamUserTemplate, user_template_params, UserProperties(**user_template_properties), - [aws_account_map[user_ref["account_id"]] for user_ref in user_refs], + list(aws_account_map.values()), ) diff --git a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py index 67b264b62..c28bc86d5 100644 --- a/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py +++ b/iambic/plugins/v0_1_0/aws/identity_center/permission_set/template_generation.py @@ -387,7 +387,7 @@ async def create_templated_permission_set( # noqa: C901 AwsIdentityCenterPermissionSetTemplate, template_params, PermissionSetProperties(**template_properties), - [aws_account_map[ps_ref["account_id"]] for ps_ref in permission_set_refs], + list(aws_account_map.values()), )