diff --git a/iambic/config/wizard.py b/iambic/config/wizard.py index ba3392ef0..1bef9b1de 100644 --- a/iambic/config/wizard.py +++ b/iambic/config/wizard.py @@ -1853,8 +1853,9 @@ def configuration_wizard_change_detection_setup(self, aws_org: AWSOrganization): return session, _ = self.get_boto3_session_for_account(aws_org.org_account_id) + # TODO: cloudtrail is not crossregion, so we need to use us-east-1 cf_client = session.client( - "cloudformation", region_name=self.aws_default_region + "cloudformation", region_name="us-east-1" # self.aws_default_region ) org_client = session.client( "organizations", region_name=self.aws_default_region @@ -1875,7 +1876,9 @@ def configuration_wizard_change_detection_setup(self, aws_org: AWSOrganization): role_name = IAMBIC_SPOKE_ROLE_NAME hub_account_id = self.hub_account_id - sqs_arn = f"arn:aws:sqs:{self.aws_default_region}:{hub_account_id}:IAMbicChangeDetectionQueue{IAMBIC_CHANGE_DETECTION_SUFFIX}" + # cloudtrail is not crossregion, so we need to use us-east-1 + # sqs_arn = f"arn:aws:sqs:{self.aws_default_region}:{hub_account_id}:IAMbicChangeDetectionQueue{IAMBIC_CHANGE_DETECTION_SUFFIX}" + sqs_arn = f"arn:aws:sqs:us-east-1:{hub_account_id}:IAMbicChangeDetectionQueue{IAMBIC_CHANGE_DETECTION_SUFFIX}" if not self.existing_role_template_map: log.info("Loading AWS role templates...") diff --git a/iambic/core/models.py b/iambic/core/models.py index 2521e3be9..c409b1d91 100644 --- a/iambic/core/models.py +++ b/iambic/core/models.py @@ -310,6 +310,7 @@ class ProposedChangeType(Enum): DELETE = "Delete" ATTACH = "Attach" DETACH = "Detach" + UNKNOWN = "Unknown" class ProposedChange(PydanticBaseModel): diff --git a/iambic/core/template_generation.py b/iambic/core/template_generation.py index c5ac44d78..29858306f 100644 --- a/iambic/core/template_generation.py +++ b/iambic/core/template_generation.py @@ -1070,7 +1070,7 @@ def merge_model( # noqa: C901 setattr(merged_model, key, new_value) else: raise TypeError( - f"Type of {type(new_value)} is not supported. {IAMBIC_ERR_MSG}" + f"Type of {type(new_value)}({key}) is not supported. {IAMBIC_ERR_MSG}" ) elif key not in iambic_fields: setattr(merged_model, key, new_value) diff --git a/iambic/core/utils.py b/iambic/core/utils.py index 9558b11e0..a3c462a1b 100644 --- a/iambic/core/utils.py +++ b/iambic/core/utils.py @@ -402,6 +402,9 @@ def evaluate_on_provider( """ from iambic.core.models import AccessModelMixin + if getattr(resource, "organization_account_needed", None): + return getattr(provider_details, "organization_account", False) + no_op_values = [IambicManaged.DISABLED] if exclude_import_only: no_op_values.append(IambicManaged.IMPORT_ONLY) diff --git a/iambic/main.py b/iambic/main.py index 12ec1f6f7..a804f9226 100644 --- a/iambic/main.py +++ b/iambic/main.py @@ -110,6 +110,7 @@ def run_expire(templates: list[str], repo_dir: str = str(pathlib.Path.cwd())): "repo_dir", required=False, type=click.Path(exists=True), + default=os.getenv("IAMBIC_REPO_DIR"), help="The repo directory containing the templates. Example: ~/iambic-templates", ) def detect(repo_dir: str): @@ -398,7 +399,7 @@ def config_discovery(repo_dir: str): ) def import_(repo_dir: str): config_path = asyncio.run(resolve_config_template_path(repo_dir)) - config = asyncio.run(load_config(config_path)) + config: Config = asyncio.run(load_config(config_path)) check_and_update_resource_limit(config) exe_message = ExecutionMessage( execution_id=str(uuid.uuid4()), command=Command.IMPORT diff --git a/iambic/plugins/v0_1_0/aws/cloud_formation/templates/IambicSpokeRole.yml b/iambic/plugins/v0_1_0/aws/cloud_formation/templates/IambicSpokeRole.yml index 1013208d4..71784585e 100644 --- a/iambic/plugins/v0_1_0/aws/cloud_formation/templates/IambicSpokeRole.yml +++ b/iambic/plugins/v0_1_0/aws/cloud_formation/templates/IambicSpokeRole.yml @@ -59,6 +59,20 @@ Resources: - sqs:GetQueueUrl - sqs:GetQueueAttributes Resource: - - 'arn:aws:sqs:*:*:IAMbicChangeDetectionQueue' + - 'arn:aws:sqs:*:*:IAMbicChangeDetectionQueue*' + - Sid: SCPsReadWrite + Effect: Allow + Action: + - organizations:CreatePolicy + - organizations:DeletePolicy + - organizations:DescribePolicy + - organizations:UpdatePolicy + - organizations:ListPolicies + - organizations:AttachPolicy + - organizations:DetachPolicy + - organizations:TagResource + - organizations:UntagResource + - organizations:ListTagsForResource + Resource: '*' ManagedPolicyArns: - 'arn:aws:iam::aws:policy/ReadOnlyAccess' diff --git a/iambic/plugins/v0_1_0/aws/event_bridge/models.py b/iambic/plugins/v0_1_0/aws/event_bridge/models.py index 254664c47..cc83799b3 100644 --- a/iambic/plugins/v0_1_0/aws/event_bridge/models.py +++ b/iambic/plugins/v0_1_0/aws/event_bridge/models.py @@ -1,4 +1,5 @@ from __future__ import annotations +from typing import Literal, Optional from pydantic import BaseModel as PydanticBaseModel @@ -32,3 +33,36 @@ class UserMessageDetails(PydanticBaseModel): account_id: str user_name: str delete: bool + + +class SCPMessageDetails(PydanticBaseModel): + account_id: str + policy_id: str + delete: bool + event: Literal[ + "CreatePolicy", + "DeletePolicy", + "UpdatePolicy", + "AttachPolicy", + "DetachPolicy", + "TagResource", + "UntagResource", + ] + + @staticmethod + def tag_event(event, source) -> bool: + """Returns True if the event is a tag/untag event related to SCPs""" + return ( + event in ["TagResource", "UntagResource"] + and source == "organizations.amazonaws.com" + ) + + @staticmethod + def get_policy_id(request_params, response_elements) -> Optional[str]: + """Returns the policy ID from the request parameters or response elements (last one if it is a CreatePolicy event)""" + return (request_params and request_params.get("policyId", None)) or ( + response_elements + and response_elements.get("policy", {}) + .get("policySummary", {}) + .get("id", None) + ) diff --git a/iambic/plugins/v0_1_0/aws/handlers.py b/iambic/plugins/v0_1_0/aws/handlers.py index 30614ebae..44565bd36 100644 --- a/iambic/plugins/v0_1_0/aws/handlers.py +++ b/iambic/plugins/v0_1_0/aws/handlers.py @@ -27,6 +27,7 @@ ManagedPolicyMessageDetails, PermissionSetMessageDetails, RoleMessageDetails, + SCPMessageDetails, UserMessageDetails, ) from iambic.plugins.v0_1_0.aws.iam.group.models import AWS_IAM_GROUP_TEMPLATE_TYPE @@ -59,6 +60,15 @@ generate_permission_set_map, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount +from iambic.plugins.v0_1_0.aws.organizations.scp.models import AWS_SCP_POLICY_TEMPLATE +from iambic.plugins.v0_1_0.aws.organizations.scp.template_generation import ( + collect_aws_scp_policies, + generate_aws_scp_policy_templates, + get_organizations_account_map, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.utils import ( + service_control_policy_is_enabled, +) if TYPE_CHECKING: from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig @@ -78,7 +88,7 @@ async def load(config: AWSConfig) -> AWSConfig: orgs_accounts = await asyncio.gather( *[org.get_accounts() for org in config.organizations] ) - for org_accounts in orgs_accounts: + for org_accounts, org in zip(orgs_accounts, config.organizations): for account in org_accounts: if ( account_elem := config_account_idx_map.get(account.account_id) @@ -89,6 +99,15 @@ async def load(config: AWSConfig) -> AWSConfig: config.accounts[ account_elem ].identity_center_details = account.identity_center_details + + # if the account is an organization account, set the organization details + if org.org_account_id == account.account_id: + await config.accounts[ + account_elem + ].set_account_organization_details( + organization=org, + config=config, + ) else: log.warning( "Account not found in config. Account will be ignored.", @@ -396,6 +415,10 @@ async def import_aws_resources( ) ) + tasks += await import_organization_resources( + exe_message, config, base_output_dir, messages, remote_worker + ) + if not exe_message.metadata or exe_message.metadata["service"] == "iam": iam_template_map = None @@ -405,7 +428,7 @@ async def import_aws_resources( template_type="AWS::IAM.*", nested=True, ) - + config.accounts = config.accounts[0:2] tasks.append( import_service_resources( exe_message, @@ -414,8 +437,8 @@ async def import_aws_resources( "iam", [ collect_aws_roles, - collect_aws_groups, collect_aws_users, + collect_aws_groups, collect_aws_managed_policies, ], [ @@ -433,6 +456,53 @@ async def import_aws_resources( await asyncio.gather(*tasks) +async def import_organization_resources( + exe_message: ExecutionMessage, + config: AWSConfig, + base_output_dir: str, + messages: list = None, + remote_worker=None, +) -> list: + tasks = [] + if len(config.organizations) > 0: + exe_messages = await config.get_command_by_organization_account(exe_message) + scp_template_map = await get_existing_template_map( + repo_dir=base_output_dir, + template_type=AWS_SCP_POLICY_TEMPLATE, + nested=True, + ) + + for exe_msg in exe_messages: + aws_account_map: dict[ + str, AWSAccount + ] = await get_organizations_account_map(exe_msg, config) + aws_account = aws_account_map[exe_msg.provider_id] # type: ignore + org_client = await aws_account.get_boto3_client("organizations") + + if not (await service_control_policy_is_enabled(org_client)): + pass + + # this is also configured at aws config load method + await aws_account.set_account_organization_details( + await config.get_organization_from_account(exe_msg.provider_id), config + ) + tasks.append( + import_service_resources( + exe_msg, + config, + base_output_dir, + "scp", + [collect_aws_scp_policies], + [generate_aws_scp_policy_templates], + messages, + remote_worker, + scp_template_map, + ) + ) + + return tasks + + async def detect_changes( # noqa: C901 config: AWSConfig, repo_dir: str ) -> Union[str, None]: @@ -445,6 +515,7 @@ async def detect_changes( # noqa: C901 group_messages = [] managed_policy_messages = [] permission_set_messages = [] + scp_messages = [] commit_message = "Out of band changes detected.\nSummary:\n" for queue_arn in config.sqs_cloudtrail_changes_queues: @@ -510,6 +581,7 @@ async def detect_changes( # noqa: C901 if actor != identity_arn: account_id = decoded_message.get("recipientAccountId") request_params = decoded_message["requestParameters"] + response_elements = decoded_message["responseElements"] event = decoded_message["eventName"] resource_id = None resource_type = None @@ -575,6 +647,34 @@ async def detect_changes( # noqa: C901 permission_set_arn=permission_set_arn, ) ) + elif scp_policy_id := SCPMessageDetails.get_policy_id( + request_params, + response_elements, + ): + resource_id = scp_policy_id + resource_type = "SCPPolicy" + scp_messages.append( + SCPMessageDetails( + account_id=account_id, + policy_id=scp_policy_id, + delete=bool(event == "DeletePolicy"), + event=event, + ) + ) + elif SCPMessageDetails.tag_event( + event, + decoded_message["eventSource"], + ): + resource_id = request_params.get("resourceId") + resource_type = "SCPPolicy" + scp_messages.append( + SCPMessageDetails( + account_id=account_id, + policy_id=resource_id, + delete=False, + event=event, + ) + ) if resource_id: commit_message = ( @@ -598,8 +698,15 @@ async def detect_changes( # noqa: C901 collect_tasks = [] iam_template_map = None identity_center_template_map = None - - if role_messages or user_messages or group_messages or managed_policy_messages: + scp_template_map = None + + if ( + role_messages + or user_messages + or group_messages + or managed_policy_messages + or scp_messages + ): iam_template_map = await get_existing_template_map( repo_dir=repo_dir, template_type="AWS::IAM.*", @@ -613,6 +720,13 @@ async def detect_changes( # noqa: C901 nested=True, ) + if scp_messages: + scp_template_map = await get_existing_template_map( + repo_dir=repo_dir, + template_type=AWS_SCP_POLICY_TEMPLATE, + nested=True, + ) + if role_messages: collect_tasks.append( collect_aws_roles(exe_message, config, iam_template_map, role_messages) @@ -640,6 +754,22 @@ async def detect_changes( # noqa: C901 permission_set_messages, ) ) + if scp_messages: + exe_messages = await config.get_command_by_organization_account(exe_message) + + # for each execution message (by organization), collect the SCP policies. + for message in exe_messages: + if current_messages := [ + m for m in scp_messages if m.account_id == message.provider_id + ]: + collect_tasks.append( + collect_aws_scp_policies( + message, + config, + scp_template_map, + current_messages, + ) + ) if collect_tasks: await asyncio.gather(*collect_tasks) @@ -683,6 +813,23 @@ async def detect_changes( # noqa: C901 permission_set_messages, ) ) + if scp_messages: + exe_messages = await config.get_command_by_organization_account(exe_message) + + # for each execution message (by organization), collect the SCP policies. + for message in exe_messages: + if current_messages := [ + m for m in scp_messages if m.account_id == message.provider_id + ]: + tasks.append( + generate_aws_scp_policy_templates( + message, + config, + repo_dir, + scp_template_map, + current_messages, + ) + ) await asyncio.gather(*tasks) return commit_message diff --git a/iambic/plugins/v0_1_0/aws/iambic_plugin.py b/iambic/plugins/v0_1_0/aws/iambic_plugin.py index b7cd3a2cb..1408f7dc8 100644 --- a/iambic/plugins/v0_1_0/aws/iambic_plugin.py +++ b/iambic/plugins/v0_1_0/aws/iambic_plugin.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, validator from iambic.core.iambic_plugin import ProviderPlugin +from iambic.core.models import ExecutionMessage from iambic.plugins.v0_1_0 import PLUGIN_VERSION from iambic.plugins.v0_1_0.aws.handlers import ( apply, @@ -23,6 +24,7 @@ AwsIdentityCenterPermissionSetTemplate, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount, AWSOrganization +from iambic.plugins.v0_1_0.aws.organizations.scp.models import AwsScpPolicyTemplate class AWSConfig(BaseModel): @@ -105,6 +107,33 @@ async def get_boto_session_from_arn(self, arn: str, region_name: str = None): aws_account = aws_account_map[account_id] return await aws_account.get_boto3_session(region_name) + async def get_organization_accounts(self) -> list[AWSAccount]: + org_accounts = [] + # each organization has an account, retrieve it + for org in self.organizations: + org_accounts += filter( + lambda acc: acc.account_id == org.org_account_id, self.accounts + ) + + return org_accounts + + async def get_organization_from_account(self, account_id) -> AWSOrganization: + """Get the organization that owns the account""" + return list( + filter(lambda org: account_id == org.org_account_id, self.organizations) + )[0] + + async def get_command_by_organization_account( + self, command: ExecutionMessage + ) -> list[ExecutionMessage]: + commands = [] + for org in self.organizations: + command_cp = command.copy() + command_cp.provider_id = org.org_account_id + commands.append(command_cp) + + return commands + IAMBIC_PLUGIN = ProviderPlugin( config_name="aws", @@ -122,5 +151,6 @@ async def get_boto_session_from_arn(self, arn: str, region_name: str = None): AwsIamRoleTemplate, AwsIamUserTemplate, AwsIamManagedPolicyTemplate, + AwsScpPolicyTemplate, ], ) diff --git a/iambic/plugins/v0_1_0/aws/models.py b/iambic/plugins/v0_1_0/aws/models.py index 2351af4f3..1d29ecb87 100644 --- a/iambic/plugins/v0_1_0/aws/models.py +++ b/iambic/plugins/v0_1_0/aws/models.py @@ -21,6 +21,8 @@ BaseModel, BaseTemplate, ExpiryModel, + ProposedChange, + ProposedChangeType, ProviderChild, TemplateChangeDetails, Variable, @@ -159,7 +161,7 @@ def resource_id(self): class BaseAWSAccountAndOrgModel(PydanticBaseModel): default_region: Optional[RegionName] = Field( - RegionName.us_east_1, + RegionName.us_west_2, description="Default region to use when making AWS requests", ) aws_profile: Optional[str] = Field( @@ -299,6 +301,11 @@ class AWSAccount(ProviderChild, BaseAWSAccountAndOrgModel): description="The role arn to assume into when making calls to the account", exclude=True, ) + organization_account: bool = Field( + False, description="if this is an organization account" + ) + organization: Optional[AWSOrganization] = Field(None) + aws_config: Optional[AWSConfig] = Field(None) class Config: fields = {"hub_session_info": {"exclude": True}} @@ -446,6 +453,18 @@ async def set_identity_center_details( group["GroupId"]: group for group in user_or_group["Groups"] } + async def set_account_organization_details( + self, + organization: AWSOrganization, + config: AWSConfig, + ): + """Set the account's organization details + when the account is the organization account + """ + self.organization_account = True + self.organization = organization + self.aws_config = config + def dict( self, *, @@ -465,6 +484,9 @@ def dict( "boto3_session_map", "hub_session_info", "identity_center_details", + "organization_account", + "organization", + "aws_config", } if exclude: exclude.update(required_exclude) @@ -781,13 +803,18 @@ async def apply(self, config: AWSConfig) -> TemplateChangeDetails: *tasks, return_exceptions=True ) proposed_changes: list[AccountChangeDetails] = [] - exceptions_seen = set() + exceptions_seen = list() for account_change in account_changes: if isinstance(account_change, AccountChangeDetails): proposed_changes.append(account_change) else: - exceptions_seen.add(str(account_change)) + exceptions_seen.append( + ProposedChange( + change_type=ProposedChangeType.UNKNOWN, + exceptions_seen=[str(account_change)], + ) # type: ignore + ) if exceptions_seen: exceptions_seen = list(exceptions_seen) @@ -803,7 +830,7 @@ async def apply(self, config: AWSConfig) -> TemplateChangeDetails: resource_id=self.resource_id, resource_type=self.resource_type, exceptions_seen=exceptions_seen, - ) + ) # type: ignore ) template_changes.extend_changes(proposed_changes) @@ -816,18 +843,20 @@ async def apply(self, config: AWSConfig) -> TemplateChangeDetails: else: cmd_verb = "detecting" log_str = f"Error encountered when {cmd_verb} resource changes." - elif account_changes and ctx.execute: - if self.deleted: - self.delete() - log_str = "Successfully removed resource." - else: - log_str = "Successfully applied resource changes." - elif account_changes: - log_str = "Successfully detected required resource changes." + log.error(log_str, accounts=relevant_accounts_str, **log_params) else: - log_str = "No changes detected for resource." + if account_changes and ctx.execute: + if self.deleted: + self.delete() + log_str = "Successfully removed resource." + else: + log_str = "Successfully applied resource changes." + elif account_changes: + log_str = "Successfully detected required resource changes." + else: + log_str = "No changes detected for resource." - log.info(log_str, accounts=relevant_accounts_str, **log_params) + log.info(log_str, accounts=relevant_accounts_str, **log_params) return template_changes @property diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/__init__.py b/iambic/plugins/v0_1_0/aws/organizations/scp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/exceptions.py b/iambic/plugins/v0_1_0/aws/organizations/scp/exceptions.py new file mode 100644 index 000000000..e39a5b53e --- /dev/null +++ b/iambic/plugins/v0_1_0/aws/organizations/scp/exceptions.py @@ -0,0 +1,2 @@ +class OrganizationAccountRequiredException(Exception): + pass diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/models.py b/iambic/plugins/v0_1_0/aws/organizations/scp/models.py new file mode 100644 index 000000000..9b7dbaf3a --- /dev/null +++ b/iambic/plugins/v0_1_0/aws/organizations/scp/models.py @@ -0,0 +1,582 @@ +from __future__ import annotations + +import asyncio +from enum import Enum +from itertools import chain +from typing import TYPE_CHECKING, Callable, List, Optional, TypedDict, Union + +from pydantic import BaseModel as PydanticBaseModel +from pydantic import Field, constr, validator + +from iambic.core.context import ctx +from iambic.core.iambic_enum import Command +from iambic.core.logger import log +from iambic.core.models import ( + AccountChangeDetails, + BaseModel, + ExpiryModel, + ProposedChange, + ProposedChangeType, +) +from iambic.core.utils import normalize_dict_keys, plugin_apply_wrapper +from iambic.plugins.v0_1_0.aws.models import ( + AccessModel, + AWSAccount, + AWSTemplate, + Description, + Tag, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.utils import ( + apply_update_policy, + apply_update_policy_tags, + apply_update_policy_targets, + create_policy, + delete_policy, + get_policy, + service_control_policy_is_enabled, +) + +if TYPE_CHECKING: + from iambic.plugins.v0_1_0.aws.iam.models import Path + from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig + from iambic.plugins.v0_1_0.aws.models import AWSOrganization + + +AWS_SCP_POLICY_TEMPLATE = "NOQ::AWS::Organizations::SCP" + + +class StatementEffect(str, Enum): + ALLOW = "Allow" + DENY = "Deny" + + +class ServiceControlPolicyTargetItemType(str, Enum): + ACCOUNT = "ACCOUNT" + ORGANIZATIONAL_UNIT = "ORGANIZATIONAL_UNIT" + ROOT = "ROOT" + + +class ServiceControlPolicyItemType(str, Enum): + SERVICE_CONTROL_POLICY = "SERVICE_CONTROL_POLICY" + TAG_POLICY = "TAG_POLICY" + BACKUP_POLICY = "BACKUP_POLICY" + AISERVICES_OPT_OUT_POLICY = "AISERVICES_OPT_OUT_POLICY" + + +class OutputModels(PydanticBaseModel): + """Output models from AWS Organizations client""" + + class Config: + use_enum_values = True + + +class PolicyStatement(AccessModel, ExpiryModel): + """ + + Note: + Unsupported elements: + - Principal + - NotPrincipal + - NotResource + """ + + effect: str = Field(..., description="Allow | Deny") + action: Optional[Union[List[str], str]] = Field( + None, + description="A single regex or list of regexes. " + "Values are the actions that can be performed on the resources in the policy statement", + example="dynamodb:list*", + ) + not_action: Optional[Union[List[str], str]] = Field( + None, + description="An advanced policy element that explicitly matches everything except the specified list of actions." + "DON'T use this with effect: allow in the same statement OR policy", + ) + resource: Optional[Union[List[str], str]] = Field( + None, + description="A single regex or list of regexes. Values specified are the resources the statement applies to", + ) + condition: Optional[dict] = Field( + None, + description="An optional set of conditions to determine of the policy applies to a resource.", + ) + sid: Optional[str] = Field( + None, + description="The Policy Statement ID.", + ) + + @property + def resource_type(self): + return "aws:organizations:service_control_policy" + + @property + def resource_id(self): + return self.sid + + +class PolicyDocument(AccessModel, ExpiryModel): + """ + + Ref: https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps_syntax.html + """ + + version: Optional[str] = Field("2012-10-17") + statement: Optional[List[PolicyStatement]] = Field( + None, + description="List of policy statements", + ) + + @property + def resource_type(self): + return "aws:policy_document" + + @property + def resource_id(self): + return self.statement[0].sid + + +class PolicyTargetProperties(BaseModel): + """ + Note: + - Root - A string that begins with “r-” followed by from 4 to 32 lowercase letters or digits. + - Account - A string that consists of exactly 12 digits. + - Organizational unit (OU) - A string that begins with “ou-” followed by from 4 to 32 lowercase letters or digits (the ID of the root + that the OU is in). This string is followed by a second “-” dash and from 8 to 32 additional lowercase letters or digits. + """ + + organizational_units: list[str] = Field(default=[], description="List of OUs ids") + accounts: list[str] = Field( + default=[], description="List of accounts (names or ids)" + ) + roots: list[str] = Field(default=[], description="List of root ids") + + @property + def resource_type(self): + return "aws:iam:scp_policy:target" + + @property + def resource_id(self): + return "|".join(self.organizational_units + self.accounts + self.roots) + + @staticmethod + def parse_targets(targets: list, config: AWSConfig): + data = dict(organizational_units=[], accounts=[], roots=[]) + + for target in targets: + key = "accounts" + target_id = target.get("target_id") + if target_id.startswith("o-") or target_id.startswith("ou-"): + key = "organizational_units" + elif target_id.startswith("r-"): + key = "roots" + else: + target_id = list( + map( + lambda a: a.account_name, + filter(lambda a: a.account_id == target_id, config.accounts), + ) + )[0] + + data[key].append(target_id) + + return data + + @staticmethod + def unparse_targets(targets: list[str], config: AWSConfig): + data = [] + + for target in targets: + if ( + target.startswith("o-") + or target.startswith("ou-") + or target.startswith("r-") + ): + target_id = target + elif target.isdigit() and len(target) == 12: + # warn: it could be possible that a person has an account name with 12 digits + target_id = target + else: + target_id = list( + map( + lambda a: a.account_id, + filter(lambda a: a.account_name == target, config.accounts), + ) + )[0] + + data.append(target_id) + + return data + + +class PolicyProperties(BaseModel): + policy_id: Optional[str] = Field( + None, + description="The ID of the policy, it is optional when creating a new policy", + required=False, + ) + policy_name: str + path: Optional[Union[str, List[Path]]] = "/" + description: Optional[Union[str, list[Description]]] = Field( + "", + description="Description of the role", + ) + type: ServiceControlPolicyItemType = Field( + default=ServiceControlPolicyItemType.SERVICE_CONTROL_POLICY + ) + aws_managed: bool = Field(False) + policy_document: Union[PolicyDocument, List[PolicyDocument]] + targets: PolicyTargetProperties = Field(default=None) + + tags: Optional[List[Tag]] = Field( + [], + description="List of tags attached to the role", + ) + + @property + def resource_type(self): + return "aws:iam:scp_policy:properties" + + @property + def resource_id(self): + return self.policy_name + + @validator("policy_document") + def sort_policy_document(cls, v: list[PolicyDocument]): + if not isinstance(v, list): + return v + sorted_v = sorted(v, key=lambda d: d.access_model_sort_weight()) + return sorted_v + + @classmethod + def sort_func(cls, attribute_name: str) -> Callable: + def _sort_func(obj): + return f"{getattr(obj, attribute_name)}!{obj.access_model_sort_weight()}" + + return _sort_func + + @validator("tags") + def sort_tags(cls, v: list[Tag]) -> list[Tag]: + sorted_v = sorted(v, key=cls.sort_func("key")) + return sorted_v + + +class AwsScpPolicyTemplate(AWSTemplate, AccessModel): + template_type = AWS_SCP_POLICY_TEMPLATE + organization_account_needed: bool = Field( + True, + description="This template needs an organization account to be applied", + ) + ARN_TEMPLATE = "arn:aws:organizations::{account_id}:policy/{organization_unit}/service_control_policy/{policy_id}" + + properties: PolicyProperties = Field( + description="The properties of the scp policy", + ) + account_id: str + org_id: str + + def get_arn(self) -> str: + return self.ARN_TEMPLATE.format( + account_id=self.account_id if not self.properties.aws_managed else "aws", + organization_unit=self.org_id, + policy_id=self.properties.policy_id, + ) + + def _apply_resource_dict(self, aws_account: AWSAccount = None) -> dict: + resource_dict = super()._apply_resource_dict(aws_account) + resource_dict["Arn"] = self.get_arn() + return resource_dict + + async def _apply_to_account(self, aws_account: AWSAccount) -> AccountChangeDetails: + if self.account_id != aws_account.account_id: + return AccountChangeDetails( + account=str(aws_account), + resource_id=self.properties.resource_id, + resource_type=self.resource_type, + new_value={}, + current_value={}, + proposed_changes=[], + exceptions_seen=[], + ) # type: ignore + + client = await aws_account.get_boto3_client("organizations") + + if not (await service_control_policy_is_enabled(client)): + log.info("Service control policy is not enabled in the organization") + return AccountChangeDetails( + account=str(aws_account), + resource_id=self.properties.resource_id, + resource_type=self.resource_type, + new_value={}, + current_value={}, + proposed_changes=[], + exceptions_seen=[], + ) # type: ignore + + account_policy = self.apply_resource_dict(aws_account) + policy_name = account_policy.get("PolicyName", "") + + account_change_details = AccountChangeDetails( + account=str(aws_account), + resource_id=self.properties.resource_id, + resource_type=self.resource_type, + new_value=dict(**account_policy), + proposed_changes=[], + exceptions_seen=[], + ) # type: ignore + + log_params = dict( + resource_type=self.resource_type, + resource_id=self.properties.resource_id, + account=str(aws_account), + ) + + current_policy = None + if account_policy.get("PolicyId"): + current_policy = await get_policy(client, account_policy.get("PolicyId")) + current_policy = current_policy.dict() + + if current_policy: + # UPDATE POLICY + account_change_details.current_value = {**current_policy} + + if ctx.command == Command.CONFIG_DISCOVERY: + # Don't overwrite a resource during config discovery + account_change_details.new_value = {} + return account_change_details + + deleted = self.get_attribute_val_for_account(aws_account, "deleted", False) + if isinstance(deleted, list): + deleted = deleted[0].deleted + + if deleted := self.is_delete_action(aws_account): + # DELETE POLICY + if current_policy: + account_change_details.new_value = None + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.DELETE, + resource_id=self.properties.resource_id, + resource_type=self.resource_type, + ) # type: ignore + ] + log_str = "Active resource found with deleted=false." + if ctx.execute: + log_str = f"{log_str} Deleting resource..." + log.debug(log_str, **log_params) + + if ctx.execute: + apply_awaitable = delete_policy( + client, account_policy.get("PolicyId"), log_params + ) + proposed_changes = await plugin_apply_wrapper( + apply_awaitable, proposed_changes + ) + + account_change_details.extend_changes(proposed_changes) + + return account_change_details + + if current_policy: + args = [ + client, + account_policy, + current_policy, + log_params, + aws_account, + ] + + tasks = [ + method(*args) + for method in ( + apply_update_policy, + apply_update_policy_targets, + apply_update_policy_tags, + ) + ] + + changes_made: list[list[ProposedChange]] = await asyncio.gather(*tasks) + if any(changes_made): + account_change_details.extend_changes( + list(chain.from_iterable(changes_made)) + ) + if current_policy.get("Name") != account_policy.get("PolicyName"): + self.identifier = account_policy["PolicyName"] + self.write() + else: + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.CREATE, + resource_id=policy_name, + resource_type=self.resource_type, + change_summary=account_policy, + ) # type: ignore + ] + log_str = "New resource found in code." + if not ctx.execute: + # Exit now because apply functions won't work if resource doesn't exist + log.debug(log_str, **log_params) + account_change_details.extend_changes(proposed_changes) + return account_change_details + + log.debug(f"{log_str} Creating resource...", **log_params) + + apply_awaitable = create_policy(client, account_policy) + account_change_details.extend_changes( + await plugin_apply_wrapper(apply_awaitable, proposed_changes) + ) + + self.properties.policy_id = account_policy.get("PolicyId") + current_policy = await get_policy(client, account_policy.get("PolicyId")) + current_policy = current_policy.dict() + + args = [client, account_policy, current_policy, log_params, aws_account] + + tasks = [apply_update_policy_targets(*args)] + + changes_made: list[list[ProposedChange]] = await asyncio.gather(*tasks) + if any(changes_made): + account_change_details.extend_changes( + list(chain.from_iterable(changes_made)) + ) + + # name and identifier must match + self.identifier = current_policy.get("Name", self.identifier) + self.write() + + self.__log_after_apply(account_change_details, log_params) + + return account_change_details + + def is_delete_action(self, aws_account): + deleted = self.get_attribute_val_for_account(aws_account, "deleted", False) + if isinstance(deleted, list): + deleted = deleted[0].deleted + + return deleted + + def __log_after_apply(self, account_change_details, log_params): + if ctx.execute and not account_change_details.exceptions_seen: + log.debug( + "Successfully finished execution on account for resource", + changes_made=bool(account_change_details.proposed_changes), + **log_params, + ) + elif account_change_details.exceptions_seen: + log.error( + "Unable to finish execution on account for resource", + exceptions_seen=[ + cd.exceptions_seen for cd in account_change_details.exceptions_seen + ], + **log_params, + ) + else: + log.debug( + "Successfully finished scanning for drift on account for resource", + requires_changes=bool(account_change_details.proposed_changes), + **log_params, + ) + + @staticmethod + def factory_template_props( + account_id: str, + policy: ServiceControlPolicyItem, + config: AWSConfig, + organization: AWSOrganization, + ): + template_params = dict( + identifier=policy.Name, + account_id=account_id, + org_id=organization.org_id, + ) + + template_properties = dict( + policy_id=policy.Id, + policy_name=policy.Name, + description=policy.Description, + type=policy.Type, + aws_managed=policy.AwsManaged, + policy_document=dict( + version=policy.PolicyDocument.Version, + statement=policy.PolicyDocument.Statement, + ), + ) + + if policy.Targets: + template_properties.update( + targets=PolicyTargetProperties.parse_obj( + PolicyTargetProperties.parse_targets( + [normalize_dict_keys(t.dict()) for t in policy.Targets], + config, + ) + ) # type: ignore + ) + + if policy.Tags: + template_properties.update( + tags=[normalize_dict_keys(t.dict()) for t in policy.Tags] # type: ignore + ) + return template_params, template_properties + + +class PolicyStatementItem(OutputModels): + Sid: Optional[str] + Effect: StatementEffect + Action: Optional[list[str] | str] + NotAction: Optional[list[str] | str] + Resource: list[str] | str + Condition: Optional[dict] + + +class PolicyDocumentItem(OutputModels): + Version: str + Statement: list[PolicyStatementItem] | PolicyStatementItem + + +class ServiceControlPolicyTargetItem(OutputModels): + TargetId: str = Field( + description=""" +Root - A string that begins with “r-” followed by from 4 to 32 lowercase +letters or digits. +Account - A string that consists of exactly 12 digits. +Organizational unit (OU) - A string that begins with “ou-” followed by +from 4 to 32 lowercase letters or digits (the ID of the root +that the OU is in). This string is followed by a second “-” dash +and from 8 to 32 additional lowercase letters or digits. + """ + ) + Arn: str + Name: str + Type: ServiceControlPolicyTargetItemType = Field( + default=ServiceControlPolicyTargetItemType.ACCOUNT + ) + + +class TagItem(OutputModels): + Key: str + Value: str + + +class ServiceControlPolicyItem(OutputModels): + Id: str + Arn: str + Name: str + Description: str + Type: ServiceControlPolicyItemType = Field( + default=ServiceControlPolicyItemType.SERVICE_CONTROL_POLICY + ) + AwsManaged: bool + Targets: list[ServiceControlPolicyTargetItem] = Field(default_factory=list) + PolicyDocument: PolicyDocumentItem + Tags: List[TagItem] + + +class ServiceControlPolicyCache(TypedDict): + file_path: str + policy_id: str + arn: str + account_id: str + + +class ServiceControlPolicyResourceFiles(TypedDict): + account_id: str + policies: list[ServiceControlPolicyCache] diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/template_generation.py b/iambic/plugins/v0_1_0/aws/organizations/scp/template_generation.py new file mode 100644 index 000000000..d4bb09f33 --- /dev/null +++ b/iambic/plugins/v0_1_0/aws/organizations/scp/template_generation.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +import asyncio +import os +from itertools import groupby +from typing import TYPE_CHECKING, Any, Optional + +import aiofiles + +from iambic.core import noq_json as json +from iambic.core.logger import log +from iambic.core.models import ExecutionMessage +from iambic.core.template_generation import ( + create_or_update_template, + delete_orphaned_templates, +) +from iambic.core.utils import NoqSemaphore, resource_file_upsert +from iambic.plugins.v0_1_0.aws.event_bridge.models import ( + SCPMessageDetails as SCPPolicyMessageDetails, +) +from iambic.plugins.v0_1_0.aws.models import AWSAccount, AWSOrganization +from iambic.plugins.v0_1_0.aws.organizations.scp.exceptions import ( + OrganizationAccountRequiredException, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + AWS_SCP_POLICY_TEMPLATE, + AwsScpPolicyTemplate, + PolicyProperties, + ServiceControlPolicyCache, + ServiceControlPolicyItem, + ServiceControlPolicyResourceFiles, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.utils import get_policy, list_policies +from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map + +if TYPE_CHECKING: + from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig + +RESOURCE_DIR = ["organizations", "scp"] + + +def get_response_dir(exe_message: ExecutionMessage, aws_account: AWSAccount) -> str: + if exe_message.provider_id: + return exe_message.get_directory(*RESOURCE_DIR) + else: + return exe_message.get_directory(aws_account.account_id, *RESOURCE_DIR) + + +def get_template_dir(base_dir: str) -> str: + return str(os.path.join(base_dir, "resources", "aws", *RESOURCE_DIR)) + + +async def generate_scp_resource_files( + exe_message: ExecutionMessage, + aws_account: AWSAccount, + filtered_policies: Optional[list[str]] = None, +) -> ServiceControlPolicyResourceFiles: + resource_dir = get_response_dir(exe_message, aws_account) + resource_file_upsert_semaphore = NoqSemaphore(resource_file_upsert, 10) + messages = [] + + response = ServiceControlPolicyResourceFiles( + account_id=aws_account.account_id, policies=[] + ) + organizations_client = await aws_account.get_boto3_client("organizations") + if filtered_policies is None: + scp_policies = await list_policies(organizations_client) + elif len(filtered_policies) > 0: + scp_policies = await asyncio.gather( + *[ + get_policy(organizations_client, policy_id) + for policy_id in set(filtered_policies) + ] + ) + else: + return response + + log.debug( + "Retrieved AWS SCP Policies.", + account_id=aws_account.account_id, + account_name=aws_account.account_name, + scp_policies_count=len(scp_policies), + ) + + for policy in scp_policies: + policy_path = os.path.join(resource_dir, f"{policy.Id}.json") + response["policies"].append( + ServiceControlPolicyCache( + file_path=policy_path, + policy_id=policy.Id, + arn=policy.Arn, + account_id=aws_account.account_id, + ) + ) + messages.append( + dict( + file_path=policy_path, + content_as_dict=policy.dict(), + replace_file=True, + ) + ) + + await resource_file_upsert_semaphore.process(messages) + log.debug( + "Finished caching AWS SCP Policies.", + account_id=aws_account.account_id, + managed_policy_count=len(scp_policies), + ) + + return response + + +async def collect_aws_scp_policies( + exe_message: ExecutionMessage, + config: AWSConfig, + scp_template_map: dict, + detect_messages: Optional[list[SCPPolicyMessageDetails | Any]] = None, +): + aws_account_map = await get_organizations_account_map(exe_message, config) + + if detect_messages: + detect_messages = [ + dm + for dm in detect_messages + if isinstance(dm, SCPPolicyMessageDetails) + and exe_message.provider_id == dm.account_id + ] + if not detect_messages: + return + + log.info( + "Generating AWS SCP templates. Beginning to retrieve AWS SCP policies.", + accounts=list(aws_account_map.keys()), + ) + + scp_policies = [] + + if detect_messages: + # Remove deleted or mark templates for update + delete_policies = set([dm.policy_id for dm in detect_messages if dm.delete]) + + if delete_policies: + existing_template_map = dict( + [ + (k, next(g)) + for k, g in groupby( + scp_template_map.get(AWS_SCP_POLICY_TEMPLATE, {}).values(), + lambda p: p.properties.policy_id, + ) + ] + ) + + for policy in delete_policies: + if existing_template := existing_template_map.get(policy): + existing_template.delete() + + generate_scp_resource_files_semaphore = NoqSemaphore( + generate_scp_resource_files, 25 + ) + + scp_policies: list[ + ServiceControlPolicyResourceFiles + ] = await generate_scp_resource_files_semaphore.process( + [ + { + "exe_message": exe_message, + "aws_account": aws_account_map[exe_message.provider_id], + "filtered_policies": set( + [ + dm.policy_id + for dm in detect_messages + if not dm.delete and dm.policy_id not in delete_policies + ] + ), + } + ] + ) + + else: + generate_scp_resource_files_semaphore = NoqSemaphore( + generate_scp_resource_files, 25 + ) + + scp_policies: list[ + ServiceControlPolicyResourceFiles + ] = await generate_scp_resource_files_semaphore.process( + [ + { + "exe_message": exe_message, + "aws_account": aws_account_map[exe_message.provider_id], # type: ignore + } + ] + ) + + log.info( + "Finished retrieving AWS SCP policy details", + accounts=list(aws_account_map.keys()), + ) + + with open( + exe_message.get_file_path( + *RESOURCE_DIR, + file_name_and_extension=f"output-{exe_message.provider_id}.json", + ), + "w", + ) as f: + f.write(json.dumps(scp_policies)) + + +async def generate_aws_scp_policy_templates( + exe_message: ExecutionMessage, + config: AWSConfig, + base_output_dir: str, + scp_template_map: dict, + detect_messages: Optional[list[SCPPolicyMessageDetails | Any]] = None, +): + """Generate AWS SCP policy templates + + Note: + this function is executed for each organization account (provider_id) + thus, each output file has one item in the list (scp_policies) + """ + + aws_account_map = await get_organizations_account_map(exe_message, config) + + existing_template_map = scp_template_map.get(AWS_SCP_POLICY_TEMPLATE, {}) + resource_dir = get_template_dir(base_output_dir) + + scp_policies: list[ + ServiceControlPolicyResourceFiles + ] = await exe_message.get_sub_exe_files( + *RESOURCE_DIR, + file_name_and_extension=f"output-{exe_message.provider_id}.json", + flatten_results=True, + ) # type: ignore + + policies: list[ServiceControlPolicyCache | dict] = [] + account_id: str + account_id, policies = scp_policies[0].values() # type: ignore + + tasks = [] + organization: AWSOrganization = list( + filter(lambda o: o.org_account_id == account_id, config.organizations) + )[0] + + for policy in policies: + tasks.append( + upsert_templated_scp_policies( + aws_account_map, + account_id, + policy, # type: ignore + resource_dir, + existing_template_map, + config, + organization, + ) + ) + + log.debug("Writing templated scp policies") + + templates: list[AwsScpPolicyTemplate] = await asyncio.gather(*tasks) + + # NEVER call this if messages are passed in because all_resource_ids will only contain those resources + if not detect_messages: + # if some templates are iambic managed, they will be None + all_resource_ids = set([t.identifier for t in templates if t is not None]) + delete_orphaned_templates( + list(existing_template_map.values()), all_resource_ids + ) + + log.info("Finished templated scp policies generation") + + return + + +async def get_organizations_account_map(exe_message, config): + aws_account_map = await get_aws_account_map(config) + + # SCP policies should be retrieved just for the organization account (provider_id) + if not exe_message.provider_id or not aws_account_map.get(exe_message.provider_id): + raise OrganizationAccountRequiredException() + + aws_account_map = { + exe_message.provider_id: aws_account_map[exe_message.provider_id] + } + + return aws_account_map + + +async def upsert_templated_scp_policies( + aws_account_map: dict, + account_id: str, + policy: ServiceControlPolicyCache, # type: ignore + resource_dir: str, + existing_template_map: dict, + config, + organization: AWSOrganization, +): + async with aiofiles.open(policy.get("file_path"), mode="r") as f: + content_dict = json.loads(await f.read()) + # policy = normalize_dict_keys(content_dict) # type: ignore + policy: ServiceControlPolicyItem = ServiceControlPolicyItem.parse_obj( + content_dict + ) + + file_path = get_template_file_path( + resource_dir, + policy.Name, + [aws_account_map[account_id].account_name], + aws_account_map, + ) + + template_params, template_properties = AwsScpPolicyTemplate.factory_template_props( + account_id, + policy, + config, + organization, + ) + + return create_or_update_template( + file_path, + existing_template_map, + policy.Name, + AwsScpPolicyTemplate, + template_params, + PolicyProperties(**template_properties), + list(aws_account_map.values()), + ) + + +def get_template_file_path( + managed_policy_dir: str, + policy_name: str, + included_accounts: list[str], + account_map: dict[str, AWSAccount], +): + if len(included_accounts) > 1: + separator = "multi_account" + elif included_accounts == ["*"] or included_accounts is None: + separator = "all_accounts" + else: + separator = included_accounts[0] + + file_name = ( + policy_name.replace(" ", "") + .replace("{{var.", "") + .replace("{{", "") + .replace("}}_", "_") + .replace("}}", "_") + .replace(".", "_") + .lower() + ) + return str(os.path.join(managed_policy_dir, separator, f"{file_name}.yaml")) diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py b/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py new file mode 100644 index 000000000..4bad838c3 --- /dev/null +++ b/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py @@ -0,0 +1,572 @@ +from __future__ import annotations + +import asyncio +import json +import random +from itertools import chain + +from deepdiff import DeepDiff +from git import TYPE_CHECKING + +from iambic.core.context import ctx +from iambic.core.logger import log +from iambic.core.models import ProposedChange, ProposedChangeType +from iambic.core.utils import NoqSemaphore, aio_wrapper, plugin_apply_wrapper +from iambic.plugins.v0_1_0.aws.utils import boto_crud_call, legacy_paginated_search + +if TYPE_CHECKING: + from iambic.plugins.v0_1_0.aws.models import AWSAccount + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + ServiceControlPolicyItem, + ServiceControlPolicyTargetItem, + ) + + +async def list_policies( + client, filter="SERVICE_CONTROL_POLICY" +) -> list[ServiceControlPolicyItem]: + """Retrieves the list of all policies in an organization of a specified type.""" + + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + ServiceControlPolicyItem, + ) + + scp_policies = await legacy_paginated_search( + client.list_policies, + response_key="Policies", + **dict(Filter=filter), + ) + + scp_policies = [p for p in scp_policies if p["AwsManaged"] is False] + + list_targets_for_policy_semaphore = NoqSemaphore(list_targets_for_policy, 10) + describe_policy_semaphore = NoqSemaphore(get_policy_statements, 10) + list_tags_by_policy_semaphore = NoqSemaphore(list_tags_by_policy, 10) + + # CHECK: come in order by scp_policies ? + targets = await list_targets_for_policy_semaphore.process( + [{"client": client, "policyId": policy["Id"]} for policy in scp_policies] + ) + + statements = await describe_policy_semaphore.process( + [{"client": client, "policyId": policy["Id"]} for policy in scp_policies] + ) + + tags = await list_tags_by_policy_semaphore.process( + [{"client": client, "policyId": policy["Id"]} for policy in scp_policies] + ) + + return [ + ServiceControlPolicyItem.parse_obj( + { + **p, + "Targets": t, + "PolicyDocument": s, + "Tags": tg, + }, + ) + for p, t, s, tg in zip(scp_policies, targets, statements, tags) + ] + + +async def list_targets_for_policy( + client, policyId: str +) -> list[ServiceControlPolicyTargetItem]: + """ + Lists all the roots, organizational units (OUs), + and accounts that the specified policy is attached to. + """ + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + ServiceControlPolicyTargetItem, + ) + + targets = await legacy_paginated_search( + client.list_targets_for_policy, + response_key="Targets", + **dict(PolicyId=policyId), + ) + + return [ServiceControlPolicyTargetItem.parse_obj(t) for t in targets] + + +async def get_policy_statements(client, policyId: str): + policy = await describe_policy(client, policyId) + return policy.get("Content", {}) + + +async def describe_policy(client, policyId: str): + from iambic.plugins.v0_1_0.aws.organizations.scp.models import PolicyDocumentItem + + res = await boto_crud_call(client.describe_policy, PolicyId=policyId) + return dict( + PolicySummary=res.get("Policy", {}).get("PolicySummary", {}), + Content=PolicyDocumentItem.parse_obj( + json.loads(res.get("Policy", {}).get("Content", {})) + ), + ) + + +async def list_tags_by_policy(client, policyId: str) -> list[dict]: + """ + Lists tags that are attached to the specified resource. + """ + targets = await legacy_paginated_search( + client.list_tags_for_resource, + response_key="Tags", + **dict(ResourceId=policyId), + ) + + return targets + + +async def get_policy(client, policyId: str) -> ServiceControlPolicyItem: + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + ServiceControlPolicyItem, + ) + + policy = await describe_policy(client, policyId) + + list_tags_by_policy_semaphore = NoqSemaphore(list_tags_by_policy, 10) + list_targets_for_policy_semaphore = NoqSemaphore(list_targets_for_policy, 10) + + tags = await list_tags_by_policy_semaphore.process( + [{"client": client, "policyId": policyId}] + ) + + targets = await list_targets_for_policy_semaphore.process( + [{"client": client, "policyId": policyId}] + ) + + return ServiceControlPolicyItem.parse_obj( + { + **policy.get("PolicySummary", {}), # type: ignore + "Targets": targets[0], + "PolicyDocument": policy.get("Content", {}), + "Tags": tags[0], + }, + ) + + +async def detach_policy(client, policyId, targetId): + log.info(f"Detaching policy {policyId} from {targetId}") + await boto_crud_call( + client.detach_policy, + PolicyId=policyId, + TargetId=targetId, + ) + log.info(f"Detached policy {policyId} from {targetId}") + + +async def delete_policy(client, policyId: str, *args, **kwargs): + """ + Deletes the specified policy from your organization. + Before you perform this operation, you must first detach + the policy from all organizational units (OUs), roots, and accounts. + """ + list_targets_for_policy_semaphore = NoqSemaphore(list_targets_for_policy, 10) + + targets = chain.from_iterable( + await list_targets_for_policy_semaphore.process( + [{"client": client, "policyId": policyId}] + ) + ) + + targets_tasks = [ + detach_policy(client, policyId, target.TargetId) for target in targets + ] + + await asyncio.gather(*targets_tasks) + + log.info(f"Deleting policy {policyId}") + + await boto_crud_call(client.delete_policy, PolicyId=policyId) + + log.info(f"Deleted policy {policyId}") + + +async def create_policy(client, policy): + if isinstance(policy["PolicyDocument"], dict): + policy["PolicyDocument"] = json.dumps(policy["PolicyDocument"]) + + res = await boto_crud_call( + client.create_policy, + Content=policy["PolicyDocument"], + Description=policy.get("Description", ""), + Name=policy.get("PolicyName", f"NewPolicy-{random.randint(0, 100):03d}"), + Type="SERVICE_CONTROL_POLICY", + Tags=policy.get("Tags", []), + ) + + policy.update( + PolicyId=res.get("Policy", {}).get("PolicySummary", {}).get("Id"), + PolicyName=res.get("Policy", {}).get("PolicySummary", {}).get("Name"), + ) + + return res.get("Policy", {}).get("PolicySummary", {}) + + +async def describe_organization( + client, +): + """Retrieves information about the organization that the user's account belongs to. + This operation can be called from any account in the organization. + """ + res = await boto_crud_call(client.describe_organization) + return res.get("Organization", {}) + + +async def service_control_policy_is_enabled(client): + """Check if SCPs are enabled for the organization.""" + org = await describe_organization(client) + + return ( + len( + [ + apt + for apt in org.get("AvailablePolicyTypes", []) + if apt.get("Type") == "SERVICE_CONTROL_POLICY" + and apt.get("Status") == "ENABLED" + ] + ) + > 0 + ) + + +async def apply_update_policy( + client, + policy, + current_policy, + log_params, + *args, +) -> list[ProposedChange]: + current_value = dict( + Name=current_policy.get("Name"), + Description=current_policy.get("Description"), + Content=current_policy["PolicyDocument"], + ) + + new_value = dict( + Name=policy.get("PolicyName"), + Description=policy.get("Description"), + Content=policy["PolicyDocument"], + ) + + diff = await aio_wrapper( + DeepDiff, + current_value, + new_value, + report_repetition=True, + ignore_order=True, + ) + + if not diff.get("values_changed"): + return [] + + tasks = [] + policy = policy.copy() + + if isinstance(policy["PolicyDocument"], dict): + policy["PolicyDocument"] = json.dumps(policy["PolicyDocument"]) + + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.UPDATE, + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + change_summary=diff.to_dict(), + current_value=current_value, + new_value=new_value, + ) # type: ignore + ] + + if ctx.execute: + apply_awaitable = boto_crud_call( + client.update_policy, + PolicyId=policy.get("PolicyId"), + Name=policy.get("PolicyName", f"NewPolicy-{random.randint(0, 100):03d}"), + Description=policy.get("Description", ""), + Content=policy["PolicyDocument"], + ) + + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) + + if tasks: + results: list[list[ProposedChange]] = await asyncio.gather(*tasks) + return list(chain.from_iterable(results)) + else: + return proposed_changes + + +async def apply_update_policy_targets( + client, + policy, + current_policy, + log_params, + aws_account: AWSAccount, + *args, +) -> list[ProposedChange]: + tasks = [] + response = [] + + t, r = __remove_targets( + client, + policy, + current_policy, + log_params, + aws_account, + ) + + tasks += t + response += r + + t, r = __apply_targets( + client, + policy, + current_policy, + log_params, + aws_account, + ) + + tasks += t + response += r + + if tasks: + results: list[list[ProposedChange]] = await asyncio.gather(*tasks) + return list(chain.from_iterable(results)) + else: + return response + + +async def apply_update_policy_tags( + client, + policy, + current_policy, + log_params, + *args, +) -> list[ProposedChange]: + tasks = [] + response = [] + + t, r = __remove_tags( + client, + policy, + current_policy, + log_params, + ) + + tasks += t + response += r + + t, r = __apply_tags( + client, + policy, + current_policy, + log_params, + ) + + tasks += t + response += r + + if tasks: + results: list[list[ProposedChange]] = await asyncio.gather(*tasks) + return list(chain.from_iterable(results)) + else: + return response + + +def __apply_tags( + client, + policy, + current_policy, + log_params, +): + """Apply tags to policy""" + response = [] + tasks = [] + existing_tag_map = { + tag["Key"]: tag.get("Value") for tag in current_policy.get("Tags", []) + } + + tags_to_apply = [ + tag + for tag in policy.get("Tags", []) + if tag.get("Value") != existing_tag_map.get(tag["Key"]) + ] + + if tags_to_apply: + log_str = "New tags discovered in AWS." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.ATTACH, + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + attribute="tags", + current_value=[], + new_value=tags_to_apply, + ) # type: ignore + ] + response.extend(proposed_changes) + if ctx.execute: + log_str = f"{log_str} Adding tags..." + apply_awaitable = boto_crud_call( + client.tag_resource, + ResourceId=policy.get("PolicyId"), + Tags=tags_to_apply, + ) + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) + log.info(log_str, tags=tags_to_apply, **log_params) + + return tasks, response + + +def __remove_tags(client, policy, current_policy, log_params): + """Remove tags from the policy that are not in the template.""" + response = [] + tasks = [] + template_tag_map = {tag["Key"]: tag.get("Value") for tag in policy.get("Tags", [])} + + if tags_to_remove := [ + tag["Key"] + for tag in current_policy.get("Tags", []) + if not template_tag_map.get(tag["Key"]) + ]: + log_str = "Stale tags discovered." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.DETACH, + attribute="tags", + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + current_value=tags_to_remove, + new_value=[], + change_summary={"TagKeys": tags_to_remove}, + ) # type: ignore + ] + response.extend(proposed_changes) + + if ctx.execute: + log_str = f"{log_str} Removing tags..." + apply_awaitable = boto_crud_call( + client.untag_resource, + ResourceId=policy.get("PolicyId"), + TagKeys=tags_to_remove, + ) + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) + log.info(log_str, tags=tags_to_remove, **log_params) + + return tasks, response + + +def __apply_targets( + client, + policy, + current_policy, + log_params, + aws_account: AWSAccount, +): + """Apply targets to policy.""" + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + PolicyTargetProperties, + ) + + response = [] + tasks = [] + + targets = list( + chain.from_iterable( + policy.get( + "Targets", dict(OrganizationalUnits=[], Accounts=[], Roots=[]) + ).values() + ) + ) + targets = PolicyTargetProperties.unparse_targets(targets, aws_account.aws_config) + + current_targets = list( + map(lambda t: t.get("TargetId"), current_policy.get("Targets")) + ) + + if targets_to_apply := [tag for tag in targets if tag not in current_targets]: + log_str = "New targets discovered." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.ATTACH, + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + attribute="targets", + current_value=current_targets, + new_value=current_targets + targets_to_apply, + ) # type: ignore + ] + response.extend(proposed_changes) + if ctx.execute: + log_str = f"{log_str} Adding targets..." + for target in targets_to_apply: + apply_awaitable = boto_crud_call( + client.attach_policy, + PolicyId=policy.get("PolicyId"), + TargetId=target, + ) + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) + log.info(log_str, tags=targets_to_apply, **log_params) + + return tasks, response + + +def __remove_targets( + client, + policy, + current_policy, + log_params, + aws_account: AWSAccount, +): + """Remove targets from policy that are not in the template.""" + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + PolicyTargetProperties, + ) + + response = [] + tasks = [] + + targets = list( + chain.from_iterable( + policy.get( + "Targets", dict(OrganizationalUnits=[], Accounts=[], Roots=[]) + ).values() + ) + ) + targets = PolicyTargetProperties.unparse_targets(targets, aws_account.aws_config) + + current_targets = list( + map(lambda t: t.get("TargetId"), current_policy.get("Targets")) + ) + + if targets_to_remove := [ + target for target in current_targets if target not in targets + ]: + log_str = "Stale targets discovered." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.DETACH, + attribute="targets", + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + current_value=targets_to_remove, + new_value=[], + change_summary={"Targets": targets_to_remove}, + ) # type: ignore + ] + response.extend(proposed_changes) + + if ctx.execute: + log_str = f"{log_str} Removing targets..." + + for target in targets_to_remove: + apply_awaitable = boto_crud_call( + client.detach_policy, + PolicyId=policy.get("PolicyId"), + TargetId=target, + ) + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) + log.info(log_str, tags=targets_to_remove, **log_params) + + return tasks, response diff --git a/iambic/plugins/v0_1_0/aws/utils.py b/iambic/plugins/v0_1_0/aws/utils.py index 7fa7c0a9d..36c72faf5 100644 --- a/iambic/plugins/v0_1_0/aws/utils.py +++ b/iambic/plugins/v0_1_0/aws/utils.py @@ -106,7 +106,7 @@ async def paginated_search( for response_key in response_keys: results[response_key].extend(response.get(response_key, [])) - if not response["IsTruncated"] or (max_results and len(results) >= max_results): + if not response.get("IsTruncated") or (max_results and len(results) >= max_results): return results if retain_key else results[response_key] else: search_kwargs["Marker"] = response["Marker"] diff --git a/test/plugins/v0_1_0/aws/iam/role/test_models.py b/test/plugins/v0_1_0/aws/iam/role/test_models.py index d41193bb6..5dc694b99 100644 --- a/test/plugins/v0_1_0/aws/iam/role/test_models.py +++ b/test/plugins/v0_1_0/aws/iam/role/test_models.py @@ -1,4 +1,5 @@ from __future__ import annotations +import re import pytest @@ -67,7 +68,9 @@ def test_merge_role_template_from_base_model_to_unsupported_type( assert type(new_document.properties.permissions_boundary) is float with pytest.raises( TypeError, - match=f"Type of {type(new_document.properties.permissions_boundary)} is not supported. {IAMBIC_ERR_MSG}", + match=re.escape( + f"Type of {type(new_document.properties.permissions_boundary)}(permissions_boundary) is not supported. {IAMBIC_ERR_MSG}" + ), ): _ = merge_model(new_document, existing_document, aws_accounts)