feat: add service control policies support (en-1912)

iambic/config/wizard.py

@@ -1853,8 +1853,9 @@ def configuration_wizard_change_detection_setup(self, aws_org: AWSOrganization):
             return
 
         session, _ = self.get_boto3_session_for_account(aws_org.org_account_id)
+        # cloudtrail is not crossregion, so we need to use us-east-1
         cf_client = session.client(
-            "cloudformation", region_name=self.aws_default_region
+            "cloudformation", region_name="us-east-1"  # self.aws_default_region
         )
         org_client = session.client(
             "organizations", region_name=self.aws_default_region
@@ -1875,7 +1876,9 @@ def configuration_wizard_change_detection_setup(self, aws_org: AWSOrganization):
 
         role_name = IAMBIC_SPOKE_ROLE_NAME
         hub_account_id = self.hub_account_id
-        sqs_arn = f"arn:aws:sqs:{self.aws_default_region}:{hub_account_id}:IAMbicChangeDetectionQueue{IAMBIC_CHANGE_DETECTION_SUFFIX}"
+        # cloudtrail is not crossregion, so we need to use us-east-1
+        # sqs_arn = f"arn:aws:sqs:{self.aws_default_region}:{hub_account_id}:IAMbicChangeDetectionQueue{IAMBIC_CHANGE_DETECTION_SUFFIX}"
+        sqs_arn = f"arn:aws:sqs:us-east-1:{hub_account_id}:IAMbicChangeDetectionQueue{IAMBIC_CHANGE_DETECTION_SUFFIX}"
 
         if not self.existing_role_template_map:
             log.info("Loading AWS role templates...")

iambic/core/models.py

@@ -310,6 +310,8 @@ class ProposedChangeType(Enum):
     DELETE = "Delete"
     ATTACH = "Attach"
     DETACH = "Detach"
+    # This is used for states that we don't know how to handle. e.g. Exceptions during apply time
+    UNKNOWN = "Unknown"
 
 
 class ProposedChange(PydanticBaseModel):

iambic/core/template_generation.py

@@ -1070,7 +1070,7 @@ def merge_model(  # noqa: C901
                 setattr(merged_model, key, new_value)
             else:
                 raise TypeError(
-                    f"Type of {type(new_value)} is not supported. {IAMBIC_ERR_MSG}"
+                    f"Type of {type(new_value)}({key}) is not supported. {IAMBIC_ERR_MSG}"
                 )
         elif key not in iambic_fields:
             setattr(merged_model, key, new_value)

iambic/core/utils.py

@@ -402,6 +402,9 @@ def evaluate_on_provider(
     """
     from iambic.core.models import AccessModelMixin
 
+    if getattr(resource, "organization_account_needed", None):
+        return getattr(provider_details, "organization_account", False)
+
     no_op_values = [IambicManaged.DISABLED]
     if exclude_import_only:
         no_op_values.append(IambicManaged.IMPORT_ONLY)

iambic/main.py

@@ -110,6 +110,7 @@ def run_expire(templates: list[str], repo_dir: str = str(pathlib.Path.cwd())):
     "repo_dir",
     required=False,
     type=click.Path(exists=True),
+    default=os.getenv("IAMBIC_REPO_DIR"),
     help="The repo directory containing the templates. Example: ~/iambic-templates",
 )
 def detect(repo_dir: str):
@@ -398,7 +399,7 @@ def config_discovery(repo_dir: str):
 )
 def import_(repo_dir: str):
     config_path = asyncio.run(resolve_config_template_path(repo_dir))
-    config = asyncio.run(load_config(config_path))
+    config: Config = asyncio.run(load_config(config_path))
     check_and_update_resource_limit(config)
     exe_message = ExecutionMessage(
         execution_id=str(uuid.uuid4()), command=Command.IMPORT

iambic/plugins/v0_1_0/aws/cloud_formation/templates/IambicSpokeRole.yml

@@ -59,6 +59,20 @@ Resources:
                   - sqs:GetQueueUrl
                   - sqs:GetQueueAttributes
                 Resource:
-                  - 'arn:aws:sqs:*:*:IAMbicChangeDetectionQueue'
+                  - 'arn:aws:sqs:*:*:IAMbicChangeDetectionQueue*'
+              - Sid: SCPsReadWrite
+                Effect: Allow
+                Action:
+                - organizations:CreatePolicy
+                - organizations:DeletePolicy
+                - organizations:DescribePolicy
+                - organizations:UpdatePolicy
+                - organizations:ListPolicies
+                - organizations:AttachPolicy
+                - organizations:DetachPolicy
+                - organizations:TagResource
+                - organizations:UntagResource
+                - organizations:ListTagsForResource
+                Resource: '*'
       ManagedPolicyArns:
         - 'arn:aws:iam::aws:policy/ReadOnlyAccess'

iambic/plugins/v0_1_0/aws/event_bridge/models.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+from typing import Literal, Optional
 
 from pydantic import BaseModel as PydanticBaseModel
 
@@ -32,3 +33,36 @@ class UserMessageDetails(PydanticBaseModel):
     account_id: str
     user_name: str
     delete: bool
+
+
+class SCPMessageDetails(PydanticBaseModel):
+    account_id: str
+    policy_id: str
+    delete: bool
+    event: Literal[
+        "CreatePolicy",
+        "DeletePolicy",
+        "UpdatePolicy",
+        "AttachPolicy",
+        "DetachPolicy",
+        "TagResource",
+        "UntagResource",
+    ]
+
+    @staticmethod
+    def tag_event(event, source) -> bool:
+        """Returns True if the event is a tag/untag event related to SCPs"""
+        return (
+            event in ["TagResource", "UntagResource"]
+            and source == "organizations.amazonaws.com"
+        )
+
+    @staticmethod
+    def get_policy_id(request_params, response_elements) -> Optional[str]:
+        """Returns the policy ID from the request parameters or response elements (last one if it is a CreatePolicy event)"""
+        return (request_params and request_params.get("policyId", None)) or (
+            response_elements
+            and response_elements.get("policy", {})
+            .get("policySummary", {})
+            .get("id", None)
+        )

iambic/plugins/v0_1_0/aws/handlers.py

@@ -27,6 +27,7 @@
     ManagedPolicyMessageDetails,
     PermissionSetMessageDetails,
     RoleMessageDetails,
+    SCPMessageDetails,
     UserMessageDetails,
 )
 from iambic.plugins.v0_1_0.aws.iam.group.models import AWS_IAM_GROUP_TEMPLATE_TYPE
@@ -59,6 +60,15 @@
     generate_permission_set_map,
 )
 from iambic.plugins.v0_1_0.aws.models import AWSAccount
+from iambic.plugins.v0_1_0.aws.organizations.scp.models import AWS_SCP_POLICY_TEMPLATE
+from iambic.plugins.v0_1_0.aws.organizations.scp.template_generation import (
+    collect_aws_scp_policies,
+    generate_aws_scp_policy_templates,
+    get_organizations_account_map,
+)
+from iambic.plugins.v0_1_0.aws.organizations.scp.utils import (
+    service_control_policy_is_enabled,
+)
 
 if TYPE_CHECKING:
     from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig
@@ -78,7 +88,7 @@ async def load(config: AWSConfig) -> AWSConfig:
         orgs_accounts = await asyncio.gather(
             *[org.get_accounts() for org in config.organizations]
         )
-        for org_accounts in orgs_accounts:
+        for org_accounts, org in zip(orgs_accounts, config.organizations):
             for account in org_accounts:
                 if (
                     account_elem := config_account_idx_map.get(account.account_id)
@@ -89,6 +99,15 @@ async def load(config: AWSConfig) -> AWSConfig:
                     config.accounts[
                         account_elem
                     ].identity_center_details = account.identity_center_details
+
+                    # if the account is an organization account, set the organization details
+                    if org.org_account_id == account.account_id:
+                        await config.accounts[
+                            account_elem
+                        ].set_account_organization_details(
+                            organization=org,
+                            config=config,
+                        )
                 else:
                     log.warning(
                         "Account not found in config. Account will be ignored.",
@@ -396,6 +415,10 @@ async def import_aws_resources(
             )
         )
 
+    tasks += await import_organization_resources(
+        exe_message, config, base_output_dir, messages, remote_worker
+    )
+
     if not exe_message.metadata or exe_message.metadata["service"] == "iam":
         iam_template_map = None
 
@@ -405,7 +428,7 @@ async def import_aws_resources(
                 template_type="AWS::IAM.*",
                 nested=True,
             )
-
+        config.accounts = config.accounts[0:2]
         tasks.append(
             import_service_resources(
                 exe_message,
@@ -414,8 +437,8 @@ async def import_aws_resources(
                 "iam",
                 [
                     collect_aws_roles,
-                    collect_aws_groups,
                     collect_aws_users,
+                    collect_aws_groups,
                     collect_aws_managed_policies,
                 ],
                 [
@@ -433,6 +456,54 @@ async def import_aws_resources(
     await asyncio.gather(*tasks)
 
 
+async def import_organization_resources(
+    exe_message: ExecutionMessage,
+    config: AWSConfig,
+    base_output_dir: str,
+    messages: list = None,
+    remote_worker=None,
+) -> list:
+    tasks = []
+    if not config.organizations:
+        return tasks
+    exe_messages = await config.get_command_by_organization_account(exe_message)
+    scp_template_map = await get_existing_template_map(
+        repo_dir=base_output_dir,
+        template_type=AWS_SCP_POLICY_TEMPLATE,
+        nested=True,
+    )
+
+    for exe_msg in exe_messages:
+        aws_account_map: dict[
+            str, AWSAccount
+        ] = await get_organizations_account_map(exe_msg, config)
+        aws_account = aws_account_map[exe_msg.provider_id]  # type: ignore
+        org_client = await aws_account.get_boto3_client("organizations")
+
+        if not (await service_control_policy_is_enabled(org_client)):
+            pass
+
+        # this is also configured at aws config load method
+        await aws_account.set_account_organization_details(
+            await config.get_organization_from_account(exe_msg.provider_id), config
+        )
+        tasks.append(
+            import_service_resources(
+                exe_msg,
+                config,
+                base_output_dir,
+                "scp",
+                [collect_aws_scp_policies],
+                [generate_aws_scp_policy_templates],
+                messages,
+                remote_worker,
+                scp_template_map,
+            )
+        )
+
+    return tasks
+
+
 async def detect_changes(  # noqa: C901
     config: AWSConfig, repo_dir: str
 ) -> Union[str, None]:
@@ -445,6 +516,7 @@ async def detect_changes(  # noqa: C901
     group_messages = []
     managed_policy_messages = []
     permission_set_messages = []
+    scp_messages = []
     commit_message = "Out of band changes detected.\nSummary:\n"
 
     for queue_arn in config.sqs_cloudtrail_changes_queues:
@@ -510,6 +582,7 @@ async def detect_changes(  # noqa: C901
                     if actor != identity_arn:
                         account_id = decoded_message.get("recipientAccountId")
                         request_params = decoded_message["requestParameters"]
+                        response_elements = decoded_message["responseElements"]
                         event = decoded_message["eventName"]
                         resource_id = None
                         resource_type = None
@@ -575,6 +648,34 @@ async def detect_changes(  # noqa: C901
                                     permission_set_arn=permission_set_arn,
                                 )
                             )
+                        elif scp_policy_id := SCPMessageDetails.get_policy_id(
+                            request_params,
+                            response_elements,
+                        ):
+                            resource_id = scp_policy_id
+                            resource_type = "SCPPolicy"
+                            scp_messages.append(
+                                SCPMessageDetails(
+                                    account_id=account_id,
+                                    policy_id=scp_policy_id,
+                                    delete=bool(event == "DeletePolicy"),
+                                    event=event,
+                                )
+                            )
+                        elif SCPMessageDetails.tag_event(
+                            event,
+                            decoded_message["eventSource"],
+                        ):
+                            resource_id = request_params.get("resourceId")
+                            resource_type = "SCPPolicy"
+                            scp_messages.append(
+                                SCPMessageDetails(
+                                    account_id=account_id,
+                                    policy_id=resource_id,
+                                    delete=False,
+                                    event=event,
+                                )
+                            )
 
                         if resource_id:
                             commit_message = (
@@ -598,8 +699,15 @@ async def detect_changes(  # noqa: C901
     collect_tasks = []
     iam_template_map = None
     identity_center_template_map = None
-
-    if role_messages or user_messages or group_messages or managed_policy_messages:
+    scp_template_map = None
+
+    if (
+        role_messages
+        or user_messages
+        or group_messages
+        or managed_policy_messages
+        or scp_messages
+    ):
         iam_template_map = await get_existing_template_map(
             repo_dir=repo_dir,
             template_type="AWS::IAM.*",
@@ -613,6 +721,13 @@ async def detect_changes(  # noqa: C901
             nested=True,
         )
 
+    if scp_messages:
+        scp_template_map = await get_existing_template_map(
+            repo_dir=repo_dir,
+            template_type=AWS_SCP_POLICY_TEMPLATE,
+            nested=True,
+        )
+
     if role_messages:
         collect_tasks.append(
             collect_aws_roles(exe_message, config, iam_template_map, role_messages)
@@ -640,6 +755,22 @@ async def detect_changes(  # noqa: C901
                 permission_set_messages,
             )
         )
+    if scp_messages:
+        exe_messages = await config.get_command_by_organization_account(exe_message)
+
+        # for each execution message (by organization), collect the SCP policies.
+        for message in exe_messages:
+            if current_messages := [
+                m for m in scp_messages if m.account_id == message.provider_id
+            ]:
+                collect_tasks.append(
+                    collect_aws_scp_policies(
+                        message,
+                        config,
+                        scp_template_map,
+                        current_messages,
+                    )
+                )
 
     if collect_tasks:
         await asyncio.gather(*collect_tasks)
@@ -683,6 +814,23 @@ async def detect_changes(  # noqa: C901
                     permission_set_messages,
                 )
             )
+        if scp_messages:
+            exe_messages = await config.get_command_by_organization_account(exe_message)
+
+            # for each execution message (by organization), collect the SCP policies.
+            for message in exe_messages:
+                if current_messages := [
+                    m for m in scp_messages if m.account_id == message.provider_id
+                ]:
+                    tasks.append(
+                        generate_aws_scp_policy_templates(
+                            message,
+                            config,
+                            repo_dir,
+                            scp_template_map,
+                            current_messages,
+                        )
+                    )
         await asyncio.gather(*tasks)
 
         return commit_message