From f7a4d787a82228ed98cc5b5ff8ac066447a14efd Mon Sep 17 00:00:00 2001 From: Jonathan Loscalzo Date: Thu, 4 May 2023 11:23:45 -0300 Subject: [PATCH 1/2] feat: add service control policies support (en-1912) --- .../aws/organizations/__init__.py | 0 .../aws/organizations/scp/__init__.py | 0 .../organizations/scp/test_create_template.py | 173 +++++ .../scp/test_template_expiration.py | 68 ++ .../organizations/scp/test_update_template.py | 169 +++++ .../aws/organizations/scp/utils.py | 149 ++++ iambic/config/wizard.py | 7 +- iambic/core/models.py | 2 + iambic/core/template_generation.py | 2 +- iambic/core/utils.py | 3 + iambic/main.py | 3 +- .../templates/IambicSpokeRole.yml | 16 +- .../plugins/v0_1_0/aws/event_bridge/models.py | 31 + iambic/plugins/v0_1_0/aws/handlers.py | 160 ++++- iambic/plugins/v0_1_0/aws/iambic_plugin.py | 30 + iambic/plugins/v0_1_0/aws/models.py | 66 +- .../v0_1_0/aws/organizations/scp/__init__.py | 0 .../aws/organizations/scp/exceptions.py | 5 + .../v0_1_0/aws/organizations/scp/models.py | 560 +++++++++++++++ .../organizations/scp/template_generation.py | 351 +++++++++ .../v0_1_0/aws/organizations/scp/utils.py | 673 ++++++++++++++++++ iambic/plugins/v0_1_0/aws/utils.py | 4 +- poetry.lock | 186 +---- pyproject.toml | 1 + test/core/test_utils.py | 11 + .../v0_1_0/aws/iam/role/test_models.py | 6 +- .../v0_1_0/aws/organizations/__init__.py | 0 .../v0_1_0/aws/organizations/conftest.py | 138 ++++ .../v0_1_0/aws/organizations/scp/__init__.py | 0 .../scp/test_import_resources.py | 29 + .../scp/test_template_generation.py | 322 +++++++++ .../aws/organizations/scp/test_utils.py | 356 +++++++++ test/plugins/v0_1_0/aws/test_event_bridge.py | 33 + 33 files changed, 3361 insertions(+), 193 deletions(-) create mode 100644 functional_tests/aws/organizations/__init__.py create mode 100644 functional_tests/aws/organizations/scp/__init__.py create mode 100644 functional_tests/aws/organizations/scp/test_create_template.py create mode 100644 functional_tests/aws/organizations/scp/test_template_expiration.py create mode 100644 functional_tests/aws/organizations/scp/test_update_template.py create mode 100644 functional_tests/aws/organizations/scp/utils.py create mode 100644 iambic/plugins/v0_1_0/aws/organizations/scp/__init__.py create mode 100644 iambic/plugins/v0_1_0/aws/organizations/scp/exceptions.py create mode 100644 iambic/plugins/v0_1_0/aws/organizations/scp/models.py create mode 100644 iambic/plugins/v0_1_0/aws/organizations/scp/template_generation.py create mode 100644 iambic/plugins/v0_1_0/aws/organizations/scp/utils.py create mode 100644 test/plugins/v0_1_0/aws/organizations/__init__.py create mode 100644 test/plugins/v0_1_0/aws/organizations/conftest.py create mode 100644 test/plugins/v0_1_0/aws/organizations/scp/__init__.py create mode 100644 test/plugins/v0_1_0/aws/organizations/scp/test_import_resources.py create mode 100644 test/plugins/v0_1_0/aws/organizations/scp/test_template_generation.py create mode 100644 test/plugins/v0_1_0/aws/organizations/scp/test_utils.py create mode 100644 test/plugins/v0_1_0/aws/test_event_bridge.py diff --git a/functional_tests/aws/organizations/__init__.py b/functional_tests/aws/organizations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/functional_tests/aws/organizations/scp/__init__.py b/functional_tests/aws/organizations/scp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/functional_tests/aws/organizations/scp/test_create_template.py b/functional_tests/aws/organizations/scp/test_create_template.py new file mode 100644 index 000000000..481f6e0e6 --- /dev/null +++ b/functional_tests/aws/organizations/scp/test_create_template.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from unittest import IsolatedAsyncioTestCase + +from functional_tests.conftest import IAMBIC_TEST_DETAILS +from iambic.core.models import ProposedChangeType +from iambic.plugins.v0_1_0.aws.models import Tag +from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + AwsScpPolicyTemplate, + PolicyTargetProperties, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.template_generation import ( + get_template_dir, +) + +from .utils import generate_policy_template + + +class CreatePolicyTestCase(IsolatedAsyncioTestCase): + templates: list[AwsScpPolicyTemplate] = [] + + async def asyncSetUp(self): + self.policy_dir = get_template_dir(IAMBIC_TEST_DETAILS.template_dir_path) + + self.org_account = next( + filter( + lambda acc: acc.organization_account, + IAMBIC_TEST_DETAILS.config.aws.accounts, + ) + ) + + self.org_client = await self.org_account.get_boto3_client("organizations") + + self.accounts = [ + acc + for acc in IAMBIC_TEST_DETAILS.config.aws.accounts + if acc.organization_account is False + ] + + async def asyncTearDown(self): + for template in self.templates: + template.deleted = True + await template.apply(IAMBIC_TEST_DETAILS.config.aws) + + async def test_create_template(self): + client = self.org_client + policy_template = await generate_policy_template( + IAMBIC_TEST_DETAILS.template_dir_path, + self.org_account, + ) # type: ignore + + policy_template.write() + + changes = await policy_template.apply(IAMBIC_TEST_DETAILS.config.aws) + + self.check_no_exception_seen(changes) + + policy_template = AwsScpPolicyTemplate.load(str(policy_template.file_path)) + + policy = ( + client.describe_policy(PolicyId=policy_template.properties.policy_id) + .get("Policy") + .get("PolicySummary") + ) + + self.check_policy_changes(policy_template, policy) + + self.templates.append(policy_template) + + async def test_create_template_with_targets(self): + client = self.org_client + policy_template = await generate_policy_template( + IAMBIC_TEST_DETAILS.template_dir_path, + self.org_account, + ) # type: ignore + + if not policy_template.properties.targets: + policy_template.properties.targets = PolicyTargetProperties() # type: ignore + + policy_template.properties.targets.accounts += [ + account.account_id for account in self.accounts + ] + + policy_template.write() + + changes = await policy_template.apply(IAMBIC_TEST_DETAILS.config.aws) + + self.check_no_exception_seen(changes) + + policy_template = AwsScpPolicyTemplate.load(str(policy_template.file_path)) + + policy = ( + client.describe_policy(PolicyId=policy_template.properties.policy_id) + .get("Policy") + .get("PolicySummary") + ) + + self.check_policy_changes(policy_template, policy) + self.check_targets(policy_template, IAMBIC_TEST_DETAILS.config.aws.accounts) + + self.templates.append(policy_template) + + async def test_create_template_with_tags_and_targets(self): + client = self.org_client + policy_template = await generate_policy_template( + IAMBIC_TEST_DETAILS.template_dir_path, + self.org_account, + ) # type: ignore + + if not policy_template.properties.targets: + policy_template.properties.targets = PolicyTargetProperties() # type: ignore + + policy_template.properties.targets.accounts += [ + account.account_id for account in self.accounts + ] + + if not policy_template.properties.tags: + policy_template.properties.tags = [] + + policy_template.properties.tags += [ + Tag(key="created_by", value="functional_test") # type: ignore + ] + + policy_template.write() + + changes = await policy_template.apply(IAMBIC_TEST_DETAILS.config.aws) + + self.check_no_exception_seen(changes) + + policy_template = AwsScpPolicyTemplate.load(str(policy_template.file_path)) + + policy = ( + client.describe_policy(PolicyId=policy_template.properties.policy_id) + .get("Policy") + .get("PolicySummary") + ) + + self.check_policy_changes(policy_template, policy) + self.check_targets(policy_template, IAMBIC_TEST_DETAILS.config.aws.accounts) + self.check_tags(policy_template) + + self.templates.append(policy_template) + + def check_policy_changes(self, policy_template, policy): + self.assertEquals(policy.get("Name"), policy_template.properties.policy_name) + self.assertEquals( + policy.get("Description"), policy_template.properties.description + ) + + def check_no_exception_seen(self, changes): + self.assertEquals(len(changes.exceptions_seen), 0) + self.assertEquals( + changes.proposed_changes[0].proposed_changes[0].change_type, + ProposedChangeType.CREATE, + ) + + def check_targets(self, template, accounts): + account_ids = sorted([account.account_id for account in self.accounts]) + + targets = self.org_client.list_targets_for_policy( + PolicyId=template.properties.policy_id + ).get("Targets") + + self.assertEquals( + sorted([target.get("TargetId") for target in targets]), account_ids + ) + + def check_tags(self, template): + listed_tags = self.org_client.list_tags_for_resource( + ResourceId=template.properties.policy_id + ).get("Tags") + + self.assertIn("created_by", [tag.get("Key") for tag in listed_tags]) diff --git a/functional_tests/aws/organizations/scp/test_template_expiration.py b/functional_tests/aws/organizations/scp/test_template_expiration.py new file mode 100644 index 000000000..15b768ce7 --- /dev/null +++ b/functional_tests/aws/organizations/scp/test_template_expiration.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import os +from datetime import datetime, timedelta, timezone +from unittest import IsolatedAsyncioTestCase + +from functional_tests.aws.organizations.scp.utils import generate_policy_template +from functional_tests.conftest import IAMBIC_TEST_DETAILS +from iambic.core.models import ProposedChangeType +from iambic.core.utils import remove_expired_resources +from iambic.plugins.v0_1_0.aws.organizations.scp.models import AwsScpPolicyTemplate +from iambic.plugins.v0_1_0.aws.organizations.scp.template_generation import ( + get_template_dir, +) + + +class ExpirationPolicyTestCase(IsolatedAsyncioTestCase): + templates: list[AwsScpPolicyTemplate] = [] + + async def asyncSetUp(self): + self.policy_dir = get_template_dir(IAMBIC_TEST_DETAILS.template_dir_path) + + self.org_account = next( + filter( + lambda acc: acc.organization_account, + IAMBIC_TEST_DETAILS.config.aws.accounts, + ) + ) + + self.org_client = await self.org_account.get_boto3_client("organizations") + + async def test_expire_policy_template(self): + template = await generate_policy_template( + IAMBIC_TEST_DETAILS.template_dir_path, + self.org_account, + ) # + + template.expires_at = datetime.now(timezone.utc) - timedelta(days=1) + template.write() + + self.assertFalse(template.deleted) + await remove_expired_resources( + template, template.resource_type, template.resource_id + ) + self.assertTrue(template.deleted) + + async def test_delete_policy(self): + template = await generate_policy_template( + IAMBIC_TEST_DETAILS.template_dir_path, + self.org_account, + ) # + + template.write() + + changes = await template.apply(IAMBIC_TEST_DETAILS.config.aws) + + template.deleted = True + template.write() + + changes = await template.apply(IAMBIC_TEST_DETAILS.config.aws) + + self.assertFalse(changes.proposed_changes[0].exceptions_seen) + self.assertFalse(os.path.exists(template.file_path)) + + self.assertEquals( + changes.proposed_changes[0].proposed_changes[0].change_type, + ProposedChangeType.DELETE, + ) diff --git a/functional_tests/aws/organizations/scp/test_update_template.py b/functional_tests/aws/organizations/scp/test_update_template.py new file mode 100644 index 000000000..07419f799 --- /dev/null +++ b/functional_tests/aws/organizations/scp/test_update_template.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import random +from unittest import IsolatedAsyncioTestCase + +from functional_tests.conftest import IAMBIC_TEST_DETAILS +from iambic.plugins.v0_1_0.aws.models import Tag +from iambic.plugins.v0_1_0.aws.organizations.scp.models import PolicyTargetProperties + +from .utils import generate_scp_policy_template_from_base + + +class UpdatePolicyTestCase(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.template = await generate_scp_policy_template_from_base( + IAMBIC_TEST_DETAILS.template_dir_path, create_policy=True + ) + self.policy_name = self.template.properties.policy_name + self.path = self.template.properties.path + self.all_account_ids = [ + account.account_id for account in IAMBIC_TEST_DETAILS.config.aws.accounts + ] + + self.org_account = next( + filter( + lambda acc: acc.organization_account, + IAMBIC_TEST_DETAILS.config.aws.accounts, + ) + ) + + self.org_client = await self.org_account.get_boto3_client("organizations") + + self.accounts = [ + acc + for acc in IAMBIC_TEST_DETAILS.config.aws.accounts + if acc.organization_account is False + ] + + async def asyncTearDown(self): + self.template.deleted = True + await self.template.apply(IAMBIC_TEST_DETAILS.config.aws) + + async def test_update_policy_without_attachments(self): + changes = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws) + self.assertEquals(len(changes.exceptions_seen), 0) + current_policy = self.org_client.describe_policy( + PolicyId=self.template.properties.policy_id + ) + self.assertEquals( + current_policy.get("Policy").get("PolicySummary").get("Name"), + self.template.properties.policy_name, + ) + self.assertEquals( + current_policy.get("Policy").get("PolicySummary").get("Name"), + self.template.identifier, + ) + + async def test_update_policy_with_targets(self): + account = self.attach_account() + + changes = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws) + + self.assertEquals( + len(changes.exceptions_seen), 0, f"failed due to {changes.exceptions_seen}" + ) + + self.check_attach_accounts(account) + + # detach policy from account + self.detach_account(account) + + changes = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws) + self.assertEquals(len(changes.exceptions_seen), 0) + + self.check_detach_account(account) + + async def test_update_policy_with_tags(self): + tags = self.attach_tags() + + changes = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws) + + self.assertEquals(len(changes.exceptions_seen), 0) + + self.check_attach_tags(tags) + + self.detach_tags(tags) + + changes = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws) + + self.assertEquals(len(changes.exceptions_seen), 0) + + self.check_detach_tags(tags) + + async def test_update_policy_with_attachments(self): + tags = self.attach_tags() + account = self.attach_account() + + changes = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws) + + self.assertEquals(len(changes.exceptions_seen), 0) + + self.check_attach_tags(tags) + self.check_attach_accounts(account) + + self.detach_tags(tags) + self.detach_account(account) + + changes = await self.template.apply(IAMBIC_TEST_DETAILS.config.aws) + + self.assertEquals(len(changes.exceptions_seen), 0) + + self.check_detach_tags(tags) + self.check_detach_account(account) + + def check_detach_account(self, account): + targets = self.org_client.list_targets_for_policy( + PolicyId=self.template.properties.policy_id + ).get("Targets") + self.assertNotIn( + account.account_id, [target.get("TargetId") for target in targets] + ) + + def detach_account(self, account): + self.template.properties.targets.accounts.remove(account.account_id) + + def check_attach_accounts(self, account): + targets = self.org_client.list_targets_for_policy( + PolicyId=self.template.properties.policy_id + ).get("Targets") + + self.assertIn( + account.account_id, [target.get("TargetId") for target in targets] + ) + + def attach_account(self): + account = random.choice(self.accounts) + + if not self.template.properties.targets: + self.template.properties.targets = PolicyTargetProperties() + + self.template.properties.targets.accounts.append(account.account_id) + return account + + def check_detach_tags(self, tags): + listed_tags = self.org_client.list_tags_for_resource( + ResourceId=self.template.properties.policy_id + ).get("Tags") + + self.assertNotIn(tags[0].key, [tag.get("Key") for tag in listed_tags]) + + def detach_tags(self, tags): + self.template.properties.tags = [ + tag for tag in self.template.properties.tags if tag.key not in tags[0].key + ] + + def check_attach_tags(self, tags): + listed_tags = self.org_client.list_tags_for_resource( + ResourceId=self.template.properties.policy_id + ).get("Tags") + + self.assertIn(tags[0].key, [tag.get("Key") for tag in listed_tags]) + + def attach_tags(self): + tags = [ + Tag(key="functional_test_tag", value="value"), # type: ignore + ] + + self.template.properties.tags = tags + return tags diff --git a/functional_tests/aws/organizations/scp/utils.py b/functional_tests/aws/organizations/scp/utils.py new file mode 100644 index 000000000..82e63e24b --- /dev/null +++ b/functional_tests/aws/organizations/scp/utils.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import random +import uuid + +from functional_tests.conftest import IAMBIC_TEST_DETAILS +from iambic.core.iambic_enum import Command +from iambic.core.logger import log +from iambic.core.models import ExecutionMessage +from iambic.core.template_generation import get_existing_template_map +from iambic.core.utils import gather_templates +from iambic.plugins.v0_1_0.aws.models import AWSAccount +from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + AWS_SCP_POLICY_TEMPLATE, + AwsScpPolicyTemplate, + PolicyDocument, + PolicyProperties, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.template_generation import ( + collect_aws_scp_policies, + generate_aws_scp_policy_templates, + get_template_dir, +) + +EXAMPLE_POLICY_DOCUMENT = '{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":"lex:*","Resource":"*"}}' + + +async def generate_policy_template( + repo_dir: str, + aws_account: AWSAccount, +): + """Generate a policy template for the given account + + Returns: + AwsScpPolicyTemplate: The generated policy template, + not yet written to disk and not yet applied to the account. + """ + + policy_dir = get_template_dir(repo_dir) + + policy_name = f"iambic_test_{random.randint(0, 10000)}" + policy_description = "This was created by a functional test." + policy_content = EXAMPLE_POLICY_DOCUMENT + + policy_template = AwsScpPolicyTemplate( + identifier=policy_name, + account_id=aws_account.account_id, + org_id=aws_account.organization.org_id, + file_path=f"{policy_dir}/{policy_name}.yaml", + properties=PolicyProperties.parse_obj( + dict( + policy_name=policy_name, + description=policy_description, + type="SERVICE_CONTROL_POLICY", + aws_managed=False, + policy_document=PolicyDocument.parse_raw_policy(policy_content), + ), + ), + ) # type: ignore + + policy_template.properties.path = "/iambic_test/" + + return policy_template + + +async def generate_scp_policy_template_from_base( + repo_dir: str, + create_policy: bool = False, +) -> AwsScpPolicyTemplate: + policy_dir = get_template_dir(repo_dir) + + if not create_policy: + scp_policies = await gather_templates(repo_dir, AWS_SCP_POLICY_TEMPLATE) + policy_template = AwsScpPolicyTemplate.load(random.choice(scp_policies)) + + if not scp_policies: + message = "Policies should have been generated previously (policies in this account are empty)" + log.error(message) + raise Exception(message) + + policy_template.properties.policy_name = ( + f"iambic_test_{random.randint(0, 10000)}" + ) + policy_template.identifier = policy_template.properties.policy_name + policy_template.properties.policy_id = None + policy_template.file_path = f"{policy_dir}/{policy_template.identifier}.yaml" + policy_template.properties.path = "/iambic_test/" + policy_template.write() + + log.info( + "Using scp policy as base", + managed_policy=policy_template.identifier, + ) + else: + # TODO: check create policy + org_account = next( + filter( + lambda acc: acc.organization_account, + IAMBIC_TEST_DETAILS.config.aws.accounts, + ) + ) + + policy_template = await generate_policy_template(repo_dir, org_account) + + policy_template.write() + + changes = await policy_template.apply(IAMBIC_TEST_DETAILS.config.aws) + + if changes.exceptions_seen: + log.error("Error creating policy", changes=changes) + raise Exception("Error creating policy") + + policy_template = AwsScpPolicyTemplate.load(str(policy_template.file_path)) + + log.info( + "Creating new scp policy", + managed_policy=policy_template.identifier, + ) + + return policy_template + + +async def scp_policy_full_import(detect_messages: list = None): + exe_message = ExecutionMessage( + execution_id=str(uuid.uuid4()), + command=Command.IMPORT, + provider_type="aws", + provider_id=IAMBIC_TEST_DETAILS.config.aws.organizations[0].org_account_id, + ) # type: ignore + + scp_template_map = await get_existing_template_map( + repo_dir=IAMBIC_TEST_DETAILS.template_dir_path, + template_type=AWS_SCP_POLICY_TEMPLATE, + nested=True, + ) + + await collect_aws_scp_policies( + exe_message, + IAMBIC_TEST_DETAILS.config.aws, + scp_template_map, + detect_messages, + ) + await generate_aws_scp_policy_templates( + exe_message, + IAMBIC_TEST_DETAILS.config.aws, + IAMBIC_TEST_DETAILS.template_dir_path, + scp_template_map, + detect_messages, + ) diff --git a/iambic/config/wizard.py b/iambic/config/wizard.py index 683ebbf2c..1b8886da6 100644 --- a/iambic/config/wizard.py +++ b/iambic/config/wizard.py @@ -1853,8 +1853,9 @@ def configuration_wizard_change_detection_setup(self, aws_org: AWSOrganization): return session, _ = self.get_boto3_session_for_account(aws_org.org_account_id) + # cloudtrail is not cross-region, so we need to use us-east-1 cf_client = session.client( - "cloudformation", region_name=self.aws_default_region + "cloudformation", region_name="us-east-1" # self.aws_default_region ) org_client = session.client( "organizations", region_name=self.aws_default_region @@ -1875,7 +1876,9 @@ def configuration_wizard_change_detection_setup(self, aws_org: AWSOrganization): role_name = IAMBIC_SPOKE_ROLE_NAME hub_account_id = self.hub_account_id - sqs_arn = f"arn:aws:sqs:{self.aws_default_region}:{hub_account_id}:IAMbicChangeDetectionQueue{IAMBIC_CHANGE_DETECTION_SUFFIX}" + # cloudtrail is not cross-region, so we need to use us-east-1 + # sqs_arn = f"arn:aws:sqs:{self.aws_default_region}:{hub_account_id}:IAMbicChangeDetectionQueue{IAMBIC_CHANGE_DETECTION_SUFFIX}" + sqs_arn = f"arn:aws:sqs:us-east-1:{hub_account_id}:IAMbicChangeDetectionQueue{IAMBIC_CHANGE_DETECTION_SUFFIX}" if not self.existing_role_template_map: log.info("Loading AWS role templates...") diff --git a/iambic/core/models.py b/iambic/core/models.py index 2521e3be9..849b41aca 100644 --- a/iambic/core/models.py +++ b/iambic/core/models.py @@ -310,6 +310,8 @@ class ProposedChangeType(Enum): DELETE = "Delete" ATTACH = "Attach" DETACH = "Detach" + # This is used for states that we don't know how to handle. e.g. Exceptions during apply time + UNKNOWN = "Unknown" class ProposedChange(PydanticBaseModel): diff --git a/iambic/core/template_generation.py b/iambic/core/template_generation.py index c5ac44d78..29858306f 100644 --- a/iambic/core/template_generation.py +++ b/iambic/core/template_generation.py @@ -1070,7 +1070,7 @@ def merge_model( # noqa: C901 setattr(merged_model, key, new_value) else: raise TypeError( - f"Type of {type(new_value)} is not supported. {IAMBIC_ERR_MSG}" + f"Type of {type(new_value)}({key}) is not supported. {IAMBIC_ERR_MSG}" ) elif key not in iambic_fields: setattr(merged_model, key, new_value) diff --git a/iambic/core/utils.py b/iambic/core/utils.py index 9558b11e0..a3c462a1b 100644 --- a/iambic/core/utils.py +++ b/iambic/core/utils.py @@ -402,6 +402,9 @@ def evaluate_on_provider( """ from iambic.core.models import AccessModelMixin + if getattr(resource, "organization_account_needed", None): + return getattr(provider_details, "organization_account", False) + no_op_values = [IambicManaged.DISABLED] if exclude_import_only: no_op_values.append(IambicManaged.IMPORT_ONLY) diff --git a/iambic/main.py b/iambic/main.py index 12ec1f6f7..a804f9226 100644 --- a/iambic/main.py +++ b/iambic/main.py @@ -110,6 +110,7 @@ def run_expire(templates: list[str], repo_dir: str = str(pathlib.Path.cwd())): "repo_dir", required=False, type=click.Path(exists=True), + default=os.getenv("IAMBIC_REPO_DIR"), help="The repo directory containing the templates. Example: ~/iambic-templates", ) def detect(repo_dir: str): @@ -398,7 +399,7 @@ def config_discovery(repo_dir: str): ) def import_(repo_dir: str): config_path = asyncio.run(resolve_config_template_path(repo_dir)) - config = asyncio.run(load_config(config_path)) + config: Config = asyncio.run(load_config(config_path)) check_and_update_resource_limit(config) exe_message = ExecutionMessage( execution_id=str(uuid.uuid4()), command=Command.IMPORT diff --git a/iambic/plugins/v0_1_0/aws/cloud_formation/templates/IambicSpokeRole.yml b/iambic/plugins/v0_1_0/aws/cloud_formation/templates/IambicSpokeRole.yml index 1013208d4..71784585e 100644 --- a/iambic/plugins/v0_1_0/aws/cloud_formation/templates/IambicSpokeRole.yml +++ b/iambic/plugins/v0_1_0/aws/cloud_formation/templates/IambicSpokeRole.yml @@ -59,6 +59,20 @@ Resources: - sqs:GetQueueUrl - sqs:GetQueueAttributes Resource: - - 'arn:aws:sqs:*:*:IAMbicChangeDetectionQueue' + - 'arn:aws:sqs:*:*:IAMbicChangeDetectionQueue*' + - Sid: SCPsReadWrite + Effect: Allow + Action: + - organizations:CreatePolicy + - organizations:DeletePolicy + - organizations:DescribePolicy + - organizations:UpdatePolicy + - organizations:ListPolicies + - organizations:AttachPolicy + - organizations:DetachPolicy + - organizations:TagResource + - organizations:UntagResource + - organizations:ListTagsForResource + Resource: '*' ManagedPolicyArns: - 'arn:aws:iam::aws:policy/ReadOnlyAccess' diff --git a/iambic/plugins/v0_1_0/aws/event_bridge/models.py b/iambic/plugins/v0_1_0/aws/event_bridge/models.py index 254664c47..8599388b1 100644 --- a/iambic/plugins/v0_1_0/aws/event_bridge/models.py +++ b/iambic/plugins/v0_1_0/aws/event_bridge/models.py @@ -1,6 +1,9 @@ from __future__ import annotations +from typing import Optional + from pydantic import BaseModel as PydanticBaseModel +from pydantic import Field class PermissionSetMessageDetails(PydanticBaseModel): @@ -32,3 +35,31 @@ class UserMessageDetails(PydanticBaseModel): account_id: str user_name: str delete: bool + + +class SCPMessageDetails(PydanticBaseModel): + account_id: str + policy_id: str + delete: bool + event: str = Field( + ..., + description="One of: CreatePolicy, DeletePolicy, UpdatePolicy, AttachPolicy, DetachPolicy, TagResource, UntagResource", + ) + + @staticmethod + def tag_event(event, source) -> bool: + """Returns True if the event is a tag/untag event related to SCPs""" + return ( + event in ["TagResource", "UntagResource"] + and source == "organizations.amazonaws.com" + ) + + @staticmethod + def get_policy_id(request_params, response_elements) -> Optional[str]: + """Returns the policy ID from the request parameters or response elements (last one if it is a CreatePolicy event)""" + return (request_params and request_params.get("policyId", None)) or ( + response_elements + and response_elements.get("policy", {}) + .get("policySummary", {}) + .get("id", None) + ) diff --git a/iambic/plugins/v0_1_0/aws/handlers.py b/iambic/plugins/v0_1_0/aws/handlers.py index 30614ebae..f23a374bc 100644 --- a/iambic/plugins/v0_1_0/aws/handlers.py +++ b/iambic/plugins/v0_1_0/aws/handlers.py @@ -5,7 +5,7 @@ import json import uuid from itertools import chain -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Coroutine, Union import boto3 @@ -27,6 +27,7 @@ ManagedPolicyMessageDetails, PermissionSetMessageDetails, RoleMessageDetails, + SCPMessageDetails, UserMessageDetails, ) from iambic.plugins.v0_1_0.aws.iam.group.models import AWS_IAM_GROUP_TEMPLATE_TYPE @@ -59,6 +60,15 @@ generate_permission_set_map, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount +from iambic.plugins.v0_1_0.aws.organizations.scp.models import AWS_SCP_POLICY_TEMPLATE +from iambic.plugins.v0_1_0.aws.organizations.scp.template_generation import ( + collect_aws_scp_policies, + generate_aws_scp_policy_templates, + get_organizations_account_map, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.utils import ( + service_control_policy_is_enabled, +) if TYPE_CHECKING: from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig @@ -78,7 +88,7 @@ async def load(config: AWSConfig) -> AWSConfig: orgs_accounts = await asyncio.gather( *[org.get_accounts() for org in config.organizations] ) - for org_accounts in orgs_accounts: + for org_accounts, org in zip(orgs_accounts, config.organizations): for account in org_accounts: if ( account_elem := config_account_idx_map.get(account.account_id) @@ -89,6 +99,15 @@ async def load(config: AWSConfig) -> AWSConfig: config.accounts[ account_elem ].identity_center_details = account.identity_center_details + + # if the account is an organization account, set the organization details + if org.org_account_id == account.account_id: + await config.accounts[ + account_elem + ].set_account_organization_details( + organization=org, + config=config, + ) else: log.warning( "Account not found in config. Account will be ignored.", @@ -374,7 +393,7 @@ async def import_aws_resources( messages: list = None, remote_worker=None, ): - tasks = [] + tasks: list[Coroutine] = [] if not exe_message.metadata or exe_message.metadata["service"] == "identity_center": identity_center_template_map = None @@ -396,6 +415,10 @@ async def import_aws_resources( ) ) + tasks += await import_organization_resources( + exe_message, config, base_output_dir, messages, remote_worker + ) # type: ignore + if not exe_message.metadata or exe_message.metadata["service"] == "iam": iam_template_map = None @@ -414,8 +437,8 @@ async def import_aws_resources( "iam", [ collect_aws_roles, - collect_aws_groups, collect_aws_users, + collect_aws_groups, collect_aws_managed_policies, ], [ @@ -433,6 +456,54 @@ async def import_aws_resources( await asyncio.gather(*tasks) +async def import_organization_resources( + exe_message: ExecutionMessage, + config: AWSConfig, + base_output_dir: str, + messages: list = None, + remote_worker=None, +) -> list[Coroutine]: + tasks = [] + if not config.organizations: + return tasks + exe_messages = await config.get_command_by_organization_account(exe_message) + scp_template_map = await get_existing_template_map( + repo_dir=base_output_dir, + template_type=AWS_SCP_POLICY_TEMPLATE, + nested=True, + ) + + for exe_msg in exe_messages: + aws_account_map: dict[str, AWSAccount] = await get_organizations_account_map( + exe_msg, config + ) + aws_account = aws_account_map[exe_msg.provider_id] # type: ignore + org_client = await aws_account.get_boto3_client("organizations") + + if not (await service_control_policy_is_enabled(org_client)): + pass + + # this is also configured at aws config load method + await aws_account.set_account_organization_details( + await config.get_organization_from_account(exe_msg.provider_id), config + ) + tasks.append( + import_service_resources( + exe_msg, + config, + base_output_dir, + "scp", + [collect_aws_scp_policies], + [generate_aws_scp_policy_templates], + messages, + remote_worker, + scp_template_map, + ) + ) + + return tasks + + async def detect_changes( # noqa: C901 config: AWSConfig, repo_dir: str ) -> Union[str, None]: @@ -445,6 +516,7 @@ async def detect_changes( # noqa: C901 group_messages = [] managed_policy_messages = [] permission_set_messages = [] + scp_messages = [] commit_message = "Out of band changes detected.\nSummary:\n" for queue_arn in config.sqs_cloudtrail_changes_queues: @@ -510,6 +582,7 @@ async def detect_changes( # noqa: C901 if actor != identity_arn: account_id = decoded_message.get("recipientAccountId") request_params = decoded_message["requestParameters"] + response_elements = decoded_message["responseElements"] event = decoded_message["eventName"] resource_id = None resource_type = None @@ -575,6 +648,34 @@ async def detect_changes( # noqa: C901 permission_set_arn=permission_set_arn, ) ) + elif scp_policy_id := SCPMessageDetails.get_policy_id( + request_params, + response_elements, + ): + resource_id = scp_policy_id + resource_type = "SCPPolicy" + scp_messages.append( + SCPMessageDetails( + account_id=account_id, + policy_id=scp_policy_id, + delete=bool(event == "DeletePolicy"), + event=event, + ) + ) + elif SCPMessageDetails.tag_event( + event, + decoded_message["eventSource"], + ): + resource_id = request_params.get("resourceId") + resource_type = "SCPPolicy" + scp_messages.append( + SCPMessageDetails( + account_id=account_id, + policy_id=resource_id, + delete=False, + event=event, + ) + ) if resource_id: commit_message = ( @@ -598,8 +699,15 @@ async def detect_changes( # noqa: C901 collect_tasks = [] iam_template_map = None identity_center_template_map = None - - if role_messages or user_messages or group_messages or managed_policy_messages: + scp_template_map = None + + if ( + role_messages + or user_messages + or group_messages + or managed_policy_messages + or scp_messages + ): iam_template_map = await get_existing_template_map( repo_dir=repo_dir, template_type="AWS::IAM.*", @@ -613,6 +721,13 @@ async def detect_changes( # noqa: C901 nested=True, ) + if scp_messages: + scp_template_map = await get_existing_template_map( + repo_dir=repo_dir, + template_type=AWS_SCP_POLICY_TEMPLATE, + nested=True, + ) + if role_messages: collect_tasks.append( collect_aws_roles(exe_message, config, iam_template_map, role_messages) @@ -640,6 +755,22 @@ async def detect_changes( # noqa: C901 permission_set_messages, ) ) + if scp_messages: + exe_messages = await config.get_command_by_organization_account(exe_message) + + # for each execution message (by organization), collect the SCP policies. + for message in exe_messages: + if current_messages := [ + m for m in scp_messages if m.account_id == message.provider_id + ]: + collect_tasks.append( + collect_aws_scp_policies( + message, + config, + scp_template_map, + current_messages, + ) + ) if collect_tasks: await asyncio.gather(*collect_tasks) @@ -683,6 +814,23 @@ async def detect_changes( # noqa: C901 permission_set_messages, ) ) + if scp_messages: + exe_messages = await config.get_command_by_organization_account(exe_message) + + # for each execution message (by organization), collect the SCP policies. + for message in exe_messages: + if current_messages := [ + m for m in scp_messages if m.account_id == message.provider_id + ]: + tasks.append( + generate_aws_scp_policy_templates( + message, + config, + repo_dir, + scp_template_map, + current_messages, + ) + ) await asyncio.gather(*tasks) return commit_message diff --git a/iambic/plugins/v0_1_0/aws/iambic_plugin.py b/iambic/plugins/v0_1_0/aws/iambic_plugin.py index b7cd3a2cb..1408f7dc8 100644 --- a/iambic/plugins/v0_1_0/aws/iambic_plugin.py +++ b/iambic/plugins/v0_1_0/aws/iambic_plugin.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, validator from iambic.core.iambic_plugin import ProviderPlugin +from iambic.core.models import ExecutionMessage from iambic.plugins.v0_1_0 import PLUGIN_VERSION from iambic.plugins.v0_1_0.aws.handlers import ( apply, @@ -23,6 +24,7 @@ AwsIdentityCenterPermissionSetTemplate, ) from iambic.plugins.v0_1_0.aws.models import AWSAccount, AWSOrganization +from iambic.plugins.v0_1_0.aws.organizations.scp.models import AwsScpPolicyTemplate class AWSConfig(BaseModel): @@ -105,6 +107,33 @@ async def get_boto_session_from_arn(self, arn: str, region_name: str = None): aws_account = aws_account_map[account_id] return await aws_account.get_boto3_session(region_name) + async def get_organization_accounts(self) -> list[AWSAccount]: + org_accounts = [] + # each organization has an account, retrieve it + for org in self.organizations: + org_accounts += filter( + lambda acc: acc.account_id == org.org_account_id, self.accounts + ) + + return org_accounts + + async def get_organization_from_account(self, account_id) -> AWSOrganization: + """Get the organization that owns the account""" + return list( + filter(lambda org: account_id == org.org_account_id, self.organizations) + )[0] + + async def get_command_by_organization_account( + self, command: ExecutionMessage + ) -> list[ExecutionMessage]: + commands = [] + for org in self.organizations: + command_cp = command.copy() + command_cp.provider_id = org.org_account_id + commands.append(command_cp) + + return commands + IAMBIC_PLUGIN = ProviderPlugin( config_name="aws", @@ -122,5 +151,6 @@ async def get_boto_session_from_arn(self, arn: str, region_name: str = None): AwsIamRoleTemplate, AwsIamUserTemplate, AwsIamManagedPolicyTemplate, + AwsScpPolicyTemplate, ], ) diff --git a/iambic/plugins/v0_1_0/aws/models.py b/iambic/plugins/v0_1_0/aws/models.py index 2351af4f3..006a366c4 100644 --- a/iambic/plugins/v0_1_0/aws/models.py +++ b/iambic/plugins/v0_1_0/aws/models.py @@ -21,6 +21,8 @@ BaseModel, BaseTemplate, ExpiryModel, + ProposedChange, + ProposedChangeType, ProviderChild, TemplateChangeDetails, Variable, @@ -64,6 +66,11 @@ def get_spoke_role_arn(account_id: str, role_name=None) -> str: return f"arn:aws:iam::{account_id}:role/{role_name}" +class StatementEffect(str, Enum): + ALLOW = "Allow" + DENY = "Deny" + + @yaml_object(yaml) class Partition(Enum): AWS = "aws" @@ -299,6 +306,13 @@ class AWSAccount(ProviderChild, BaseAWSAccountAndOrgModel): description="The role arn to assume into when making calls to the account", exclude=True, ) + organization: Optional[AWSOrganization] = Field( + None, description="The AWS Organization this account belongs to" + ) + aws_config: Optional[AWSConfig] = Field( + None, + description="when an account is an organization account, it needs the AWS Config settings", + ) class Config: fields = {"hub_session_info": {"exclude": True}} @@ -446,6 +460,17 @@ async def set_identity_center_details( group["GroupId"]: group for group in user_or_group["Groups"] } + async def set_account_organization_details( + self, + organization: AWSOrganization, + config: AWSConfig, + ): + """Set the account's organization details + when the account is the organization account + """ + self.organization = organization + self.aws_config = config + def dict( self, *, @@ -465,6 +490,9 @@ def dict( "boto3_session_map", "hub_session_info", "identity_center_details", + "organization_account", + "organization", + "aws_config", } if exclude: exclude.update(required_exclude) @@ -517,6 +545,11 @@ def all_identifiers(self) -> set[str]: else: return set(self.account_id) + @property + def organization_account(self) -> bool: + """if current account is an organization account""" + return bool(self.organization and self.aws_config) + def __str__(self): return f"{self.account_name} - ({self.account_id})" @@ -781,13 +814,18 @@ async def apply(self, config: AWSConfig) -> TemplateChangeDetails: *tasks, return_exceptions=True ) proposed_changes: list[AccountChangeDetails] = [] - exceptions_seen = set() + exceptions_seen = list() for account_change in account_changes: if isinstance(account_change, AccountChangeDetails): proposed_changes.append(account_change) else: - exceptions_seen.add(str(account_change)) + exceptions_seen.append( + ProposedChange( + change_type=ProposedChangeType.UNKNOWN, + exceptions_seen=[str(account_change)], + ) # type: ignore + ) if exceptions_seen: exceptions_seen = list(exceptions_seen) @@ -803,7 +841,7 @@ async def apply(self, config: AWSConfig) -> TemplateChangeDetails: resource_id=self.resource_id, resource_type=self.resource_type, exceptions_seen=exceptions_seen, - ) + ) # type: ignore ) template_changes.extend_changes(proposed_changes) @@ -816,18 +854,20 @@ async def apply(self, config: AWSConfig) -> TemplateChangeDetails: else: cmd_verb = "detecting" log_str = f"Error encountered when {cmd_verb} resource changes." - elif account_changes and ctx.execute: - if self.deleted: - self.delete() - log_str = "Successfully removed resource." - else: - log_str = "Successfully applied resource changes." - elif account_changes: - log_str = "Successfully detected required resource changes." + log.error(log_str, accounts=relevant_accounts_str, **log_params) else: - log_str = "No changes detected for resource." + if account_changes and ctx.execute: + if self.deleted: + self.delete() + log_str = "Successfully removed resource." + else: + log_str = "Successfully applied resource changes." + elif account_changes: + log_str = "Successfully detected required resource changes." + else: + log_str = "No changes detected for resource." - log.info(log_str, accounts=relevant_accounts_str, **log_params) + log.info(log_str, accounts=relevant_accounts_str, **log_params) return template_changes @property diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/__init__.py b/iambic/plugins/v0_1_0/aws/organizations/scp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/exceptions.py b/iambic/plugins/v0_1_0/aws/organizations/scp/exceptions.py new file mode 100644 index 000000000..7bd373468 --- /dev/null +++ b/iambic/plugins/v0_1_0/aws/organizations/scp/exceptions.py @@ -0,0 +1,5 @@ +from __future__ import annotations + + +class OrganizationAccountRequiredException(Exception): + pass diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/models.py b/iambic/plugins/v0_1_0/aws/organizations/scp/models.py new file mode 100644 index 000000000..9bbcbe21c --- /dev/null +++ b/iambic/plugins/v0_1_0/aws/organizations/scp/models.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import asyncio +import json +from enum import Enum +from itertools import chain +from typing import TYPE_CHECKING, Callable, List, Optional, TypedDict, Union + +from pydantic import BaseModel as PydanticBaseModel +from pydantic import Field, validator + +from iambic.core.context import ctx +from iambic.core.iambic_enum import Command +from iambic.core.logger import log +from iambic.core.models import ( + AccountChangeDetails, + BaseModel, + ExpiryModel, + ProposedChange, + ProposedChangeType, +) +from iambic.core.utils import normalize_dict_keys, plugin_apply_wrapper +from iambic.plugins.v0_1_0.aws.iam.models import Path +from iambic.plugins.v0_1_0.aws.iam.policy.models import PolicyStatement +from iambic.plugins.v0_1_0.aws.models import ( + AccessModel, + AWSAccount, + AWSTemplate, + Description, + StatementEffect, + Tag, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.utils import ( + apply_update_policy, + apply_update_policy_tags, + apply_update_policy_targets, + create_policy, + delete_policy, + get_policy, + service_control_policy_is_enabled, +) + +if TYPE_CHECKING: + from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig + from iambic.plugins.v0_1_0.aws.models import AWSOrganization + + +AWS_SCP_POLICY_TEMPLATE = "NOQ::AWS::Organizations::SCP" + + +class ServiceControlPolicyTargetItemType(str, Enum): + ACCOUNT = "ACCOUNT" + ORGANIZATIONAL_UNIT = "ORGANIZATIONAL_UNIT" + ROOT = "ROOT" + + +class OrganizationsPolicyType(str, Enum): + """ + AWS Organizations supports the following policy types. You specify the policy type when you create a policy. + + Possible values: + - TAG_POLICY + - BACKUP_POLICY + - AISERVICES_OPT_OUT_POLICY + + Ref: + - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/organizations/client/list_policies.html + """ + + SERVICE_CONTROL_POLICY = "SERVICE_CONTROL_POLICY" + TAG_POLICY = "TAG_POLICY" + BACKUP_POLICY = "BACKUP_POLICY" + AISERVICES_OPT_OUT_POLICY = "AISERVICES_OPT_OUT_POLICY" + + +class OutputModels(PydanticBaseModel): + """Response models from AWS Organizations client""" + + class Config: + use_enum_values = True + + +class PolicyTargetProperties(BaseModel): + """ + Note: + - Root - A string that begins with “r-” followed by from 4 to 32 lowercase letters or digits. + - Account - A string that consists of exactly 12 digits. + - Organizational unit (OU) - A string that begins with “ou-” followed by from 4 to 32 lowercase letters or digits (the ID of the root + that the OU is in). This string is followed by a second “-” dash and from 8 to 32 additional lowercase letters or digits. + """ + + organizational_units: list[str] = Field(default=[], description="List of OUs ids") + accounts: list[str] = Field( + default=[], description="List of accounts (names or ids)" + ) + roots: list[str] = Field(default=[], description="List of root ids") + + @property + def resource_type(self): + return "aws:iam:scp_policy:target" + + @property + def resource_id(self): + return "|".join(self.organizational_units + self.accounts + self.roots) + + @staticmethod + def parse_targets(targets: list, config: AWSConfig): + data = dict(organizational_units=[], accounts=[], roots=[]) + + for target in targets: + key = "accounts" + target_id = target.get("target_id") + if target_id.startswith("o-") or target_id.startswith("ou-"): + key = "organizational_units" + elif target_id.startswith("r-"): + key = "roots" + else: + target_id = list( + map( + lambda a: a.account_name, + filter(lambda a: a.account_id == target_id, config.accounts), + ) + )[0] + + data[key].append(target_id) + + return data + + @staticmethod + def unparse_targets(targets: list[str], config: AWSConfig): + data = [] + + for target in targets: + if ( + target.startswith("o-") + or target.startswith("ou-") + or target.startswith("r-") + ): + target_id = target + elif target.isdigit() and len(target) == 12: + # warn: it could be possible that a person has an account name with 12 digits + target_id = target + else: + target_id = list( + map( + lambda a: a.account_id, + filter(lambda a: a.account_name == target, config.accounts), + ) + )[0] + + data.append(target_id) + + return data + + +class PolicyDocument(AccessModel, ExpiryModel): + version: str = "2012-10-17" + statement: Union[List[PolicyStatement], PolicyStatement] = Field( + ..., + description="List of policy statements", + ) + + @property + def resource_type(self): + return "aws:policy_document" + + @property + def resource_id(self): + return self.statement.sid + + @staticmethod + def parse_raw_policy(resource_raw) -> "PolicyDocument": + resource = json.loads(resource_raw) + resource = normalize_dict_keys(resource) + return PolicyDocument(**resource) # type: ignore + + @validator("statement") + def validate_statement(cls, statements): + keys = ["principal", "not_principal", "not_resource"] + for key in keys: + for statement in statements: + if getattr(statement, key, None): + raise ValueError(f"{key} is not supported") + return statements + + +class PolicyProperties(BaseModel): + policy_id: Optional[str] = Field( + None, + description="The ID of the policy, it is optional when creating a new policy", + required=False, + ) + policy_name: str + path: Optional[Union[str, List["Path"]]] = "/" + description: Optional[Union[str, list[Description]]] = Field( + "", + description="Description of the role", + ) + type: OrganizationsPolicyType = Field( + default=OrganizationsPolicyType.SERVICE_CONTROL_POLICY + ) + aws_managed: bool = Field(False) + policy_document: Union[PolicyDocument, List[PolicyDocument]] = Field( + ..., + description="Policy document, Unsupported elements: Principal, NotPrincipal, NotResource", + ) + targets: PolicyTargetProperties = Field(default=None) + + tags: Optional[List[Tag]] = Field( + [], + description="List of tags attached to the role", + ) + + @property + def resource_type(self): + return "aws:iam:scp_policy:properties" + + @property + def resource_id(self): + return self.policy_name + + @validator("policy_document") + def sort_policy_document(cls, v: list[PolicyDocument]): + if not isinstance(v, list): + return v + sorted_v = sorted(v, key=lambda d: d.access_model_sort_weight()) + return sorted_v + + @classmethod + def sort_func(cls, attribute_name: str) -> Callable: + def _sort_func(obj): + return f"{getattr(obj, attribute_name)}!{obj.access_model_sort_weight()}" + + return _sort_func + + @validator("tags") + def sort_tags(cls, v: list[Tag]) -> list[Tag]: + sorted_v = sorted(v, key=cls.sort_func("key")) + return sorted_v + + +class AwsScpPolicyTemplate(AWSTemplate, AccessModel): + template_type = AWS_SCP_POLICY_TEMPLATE + organization_account_needed: bool = Field( + True, + description="This template needs an organization account to be applied", + ) + ARN_TEMPLATE = "arn:aws:organizations::{account_id}:policy/{organization_unit}/service_control_policy/{policy_id}" + + properties: PolicyProperties = Field( + description="The properties of the scp policy", + ) + account_id: str + org_id: str + + def get_arn(self) -> str: + return self.ARN_TEMPLATE.format( + account_id=self.account_id if not self.properties.aws_managed else "aws", + organization_unit=self.org_id, + policy_id=self.properties.policy_id, + ) + + def _apply_resource_dict(self, aws_account: AWSAccount = None) -> dict: + resource_dict = super()._apply_resource_dict(aws_account) + resource_dict["Arn"] = self.get_arn() + return resource_dict + + async def _apply_to_account(self, aws_account: AWSAccount) -> AccountChangeDetails: + if self.account_id != aws_account.account_id: + return AccountChangeDetails( + account=str(aws_account), + resource_id=self.properties.resource_id, + resource_type=self.resource_type, + new_value={}, + current_value={}, + proposed_changes=[], + exceptions_seen=[], + ) # type: ignore + + client = await aws_account.get_boto3_client("organizations") + + if not (await service_control_policy_is_enabled(client)): + log.info("Service control policy is not enabled in the organization") + return AccountChangeDetails( + account=str(aws_account), + resource_id=self.properties.resource_id, + resource_type=self.resource_type, + new_value={}, + current_value={}, + proposed_changes=[], + exceptions_seen=[], + ) # type: ignore + + account_policy = self.apply_resource_dict(aws_account) + policy_name = account_policy.get("PolicyName", "") + + account_change_details = AccountChangeDetails( + account=str(aws_account), + resource_id=self.properties.resource_id, + resource_type=self.resource_type, + new_value=dict(**account_policy), + proposed_changes=[], + exceptions_seen=[], + ) # type: ignore + + log_params = dict( + resource_type=self.resource_type, + resource_id=self.properties.resource_id, + account=str(aws_account), + ) + + current_policy = None + if account_policy.get("PolicyId"): + current_policy = await get_policy(client, account_policy.get("PolicyId")) + current_policy = current_policy.dict() + + if current_policy: + # UPDATE POLICY + account_change_details.current_value = {**current_policy} + + if ctx.command == Command.CONFIG_DISCOVERY: + # Don't overwrite a resource during config discovery + account_change_details.new_value = {} + return account_change_details + + if self.is_delete_action(aws_account): + # DELETE POLICY + if current_policy: + account_change_details.new_value = None + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.DELETE, + resource_id=self.properties.resource_id, + resource_type=self.resource_type, + ) # type: ignore + ] + log_str = "Active resource found with deleted=false." + if ctx.execute: + log_str = f"{log_str} Deleting resource..." + log.debug(log_str, **log_params) + + if ctx.execute: + apply_awaitable = delete_policy( + client, account_policy.get("PolicyId"), log_params + ) + proposed_changes = await plugin_apply_wrapper( + apply_awaitable, proposed_changes + ) + + account_change_details.extend_changes(proposed_changes) + + return account_change_details + + if current_policy: + args = [ + client, + account_policy, + current_policy, + log_params, + aws_account, + ] + + tasks = [ + method(*args) + for method in ( + apply_update_policy, + apply_update_policy_targets, + apply_update_policy_tags, + ) + ] + + changes_made: list[list[ProposedChange]] = await asyncio.gather(*tasks) + if any(changes_made): + account_change_details.extend_changes( + list(chain.from_iterable(changes_made)) + ) + if current_policy.get("Name") != account_policy.get("PolicyName"): + self.identifier = account_policy["PolicyName"] + self.write() + else: + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.CREATE, + resource_id=policy_name, + resource_type=self.resource_type, + change_summary=account_policy, + ) # type: ignore + ] + log_str = "New resource found in code." + if not ctx.execute: + # Exit now because apply functions won't work if resource doesn't exist + log.debug(log_str, **log_params) + account_change_details.extend_changes(proposed_changes) + return account_change_details + + log.debug(f"{log_str} Creating resource...", **log_params) + + apply_awaitable = create_policy(client, account_policy) + account_change_details.extend_changes( + await plugin_apply_wrapper(apply_awaitable, proposed_changes) + ) + + self.properties.policy_id = account_policy.get("PolicyId") + current_policy = await get_policy(client, account_policy.get("PolicyId")) + current_policy = current_policy.dict() + + args = [client, account_policy, current_policy, log_params, aws_account] + + tasks = [ + apply_update_policy_tags(*args), + apply_update_policy_targets(*args), + ] + + changes_made: list[list[ProposedChange]] = await asyncio.gather(*tasks) + if any(changes_made): + account_change_details.extend_changes( + list(chain.from_iterable(changes_made)) + ) + + # name and identifier must match + self.identifier = current_policy.get("Name", self.identifier) + self.write() + + self.__log_after_apply(account_change_details, log_params) + + return account_change_details + + def is_delete_action(self, aws_account): + deleted = self.get_attribute_val_for_account(aws_account, "deleted", False) + if isinstance(deleted, list): + deleted = deleted[0].deleted + + return deleted + + def __log_after_apply(self, account_change_details, log_params): + if ctx.execute and not account_change_details.exceptions_seen: + log.debug( + "Successfully finished execution on account for resource", + changes_made=bool(account_change_details.proposed_changes), + **log_params, + ) + elif account_change_details.exceptions_seen: + log.error( + "Unable to finish execution on account for resource", + exceptions_seen=[ + cd.exceptions_seen for cd in account_change_details.exceptions_seen + ], + **log_params, + ) + else: + log.debug( + "Successfully finished scanning for drift on account for resource", + requires_changes=bool(account_change_details.proposed_changes), + **log_params, + ) + + @staticmethod + def factory_template_props( + account_id: str, + policy: ServiceControlPolicyItem, + config: AWSConfig, + organization: AWSOrganization, + ): + template_params = dict( + identifier=policy.Name, + account_id=account_id, + org_id=organization.org_id, + ) + + template_properties = dict( + policy_id=policy.Id, + policy_name=policy.Name, + description=policy.Description, + type=policy.Type, + aws_managed=policy.AwsManaged, + policy_document=dict( + version=policy.PolicyDocument.Version, + statement=policy.PolicyDocument.Statement, + ), + ) + + if policy.Targets: + template_properties.update( + targets=PolicyTargetProperties.parse_obj( + PolicyTargetProperties.parse_targets( + [normalize_dict_keys(t.dict()) for t in policy.Targets], + config, + ) + ) # type: ignore + ) + + if policy.Tags: + template_properties.update( + tags=[normalize_dict_keys(t.dict()) for t in policy.Tags] # type: ignore + ) + return template_params, template_properties + + +class PolicyStatementItem(OutputModels): + Sid: Optional[str] + Effect: StatementEffect + Action: Optional[Union[list[str], str]] + NotAction: Optional[Union[list[str], str]] + Resource: Union[list[str], str] + Condition: Optional[dict] + + +class PolicyDocumentItem(OutputModels): + Version: str + Statement: Union[list[PolicyStatementItem], PolicyStatementItem] + + +class ServiceControlPolicyTargetItem(OutputModels): + TargetId: str = Field( + description=""" +Root - A string that begins with “r-” followed by from 4 to 32 lowercase +letters or digits. +Account - A string that consists of exactly 12 digits. +Organizational unit (OU) - A string that begins with “ou-” followed by +from 4 to 32 lowercase letters or digits (the ID of the root +that the OU is in). This string is followed by a second “-” dash +and from 8 to 32 additional lowercase letters or digits. + """ + ) + Arn: str + Name: str + Type: ServiceControlPolicyTargetItemType = Field( + default=ServiceControlPolicyTargetItemType.ACCOUNT + ) + + +class TagItem(OutputModels): + Key: str + Value: str + + +class ServiceControlPolicyItem(OutputModels): + Id: str + Arn: str + Name: str + Description: str + Type: OrganizationsPolicyType = Field( + default=OrganizationsPolicyType.SERVICE_CONTROL_POLICY + ) + AwsManaged: bool + Targets: list[ServiceControlPolicyTargetItem] = Field(default_factory=list) + PolicyDocument: PolicyDocumentItem + Tags: List[TagItem] + + +class ServiceControlPolicyCache(TypedDict): + file_path: str + policy_id: str + arn: str + account_id: str + + +class ServiceControlPolicyResourceFiles(TypedDict): + account_id: str + policies: list[ServiceControlPolicyCache] diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/template_generation.py b/iambic/plugins/v0_1_0/aws/organizations/scp/template_generation.py new file mode 100644 index 000000000..ab1bfeac1 --- /dev/null +++ b/iambic/plugins/v0_1_0/aws/organizations/scp/template_generation.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +import asyncio +import os +from itertools import groupby +from typing import TYPE_CHECKING, Any, Optional, Union + +import aiofiles + +from iambic.core import noq_json as json +from iambic.core.logger import log +from iambic.core.models import ExecutionMessage +from iambic.core.template_generation import ( + create_or_update_template, + delete_orphaned_templates, +) +from iambic.core.utils import NoqSemaphore, resource_file_upsert +from iambic.plugins.v0_1_0.aws.event_bridge.models import ( + SCPMessageDetails as SCPPolicyMessageDetails, +) +from iambic.plugins.v0_1_0.aws.models import AWSAccount, AWSOrganization +from iambic.plugins.v0_1_0.aws.organizations.scp.exceptions import ( + OrganizationAccountRequiredException, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + AWS_SCP_POLICY_TEMPLATE, + AwsScpPolicyTemplate, + PolicyProperties, + ServiceControlPolicyCache, + ServiceControlPolicyItem, + ServiceControlPolicyResourceFiles, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.utils import get_policy, list_policies +from iambic.plugins.v0_1_0.aws.utils import get_aws_account_map + +if TYPE_CHECKING: + from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig + +RESOURCE_DIR = ["organizations", "scp"] + + +def get_response_dir(exe_message: ExecutionMessage, aws_account: AWSAccount) -> str: + if exe_message.provider_id: + return exe_message.get_directory(*RESOURCE_DIR) + else: + return exe_message.get_directory(aws_account.account_id, *RESOURCE_DIR) + + +def get_template_dir(base_dir: str) -> str: + return str(os.path.join(base_dir, "resources", "aws", *RESOURCE_DIR)) + + +async def generate_scp_resource_files( + exe_message: ExecutionMessage, + aws_account: AWSAccount, + filtered_policies: Optional[list[str]] = None, +) -> ServiceControlPolicyResourceFiles: + resource_dir = get_response_dir(exe_message, aws_account) + resource_file_upsert_semaphore = NoqSemaphore(resource_file_upsert, 10) + messages = [] + + response = ServiceControlPolicyResourceFiles( + account_id=aws_account.account_id, policies=[] + ) + organizations_client = await aws_account.get_boto3_client("organizations") + if filtered_policies is None: + scp_policies = await list_policies(organizations_client) + elif len(filtered_policies) > 0: + scp_policies = await asyncio.gather( + *[ + get_policy(organizations_client, policy_id) + for policy_id in set(filtered_policies) + ] + ) + else: + return response + + log.debug( + "Retrieved AWS SCP Policies.", + account_id=aws_account.account_id, + account_name=aws_account.account_name, + scp_policies_count=len(scp_policies), + ) + + for policy in scp_policies: + policy_path = os.path.join(resource_dir, f"{policy.Id}.json") + response["policies"].append( + ServiceControlPolicyCache( + file_path=policy_path, + policy_id=policy.Id, + arn=policy.Arn, + account_id=aws_account.account_id, + ) + ) + messages.append( + dict( + file_path=policy_path, + content_as_dict=policy.dict(), + replace_file=True, + ) + ) + + await resource_file_upsert_semaphore.process(messages) + log.debug( + "Finished caching AWS SCP Policies.", + account_id=aws_account.account_id, + managed_policy_count=len(scp_policies), + ) + + return response + + +async def collect_aws_scp_policies( + exe_message: ExecutionMessage, + config: AWSConfig, + scp_template_map: dict, + detect_messages: Optional[list[Union[SCPPolicyMessageDetails, Any]]] = None, +): + aws_account_map = await get_organizations_account_map(exe_message, config) + + if detect_messages: + detect_messages = [ + dm + for dm in detect_messages + if isinstance(dm, SCPPolicyMessageDetails) + and exe_message.provider_id == dm.account_id + ] + if not detect_messages: + return + + log.info( + "Generating AWS SCP templates. Beginning to retrieve AWS SCP policies.", + accounts=list(aws_account_map.keys()), + ) + + if detect_messages: + # Remove deleted or mark templates for update + delete_policies = set([dm.policy_id for dm in detect_messages if dm.delete]) + + if delete_policies: + existing_template_map = dict( + [ + (k, next(g)) + for k, g in groupby( + scp_template_map.get(AWS_SCP_POLICY_TEMPLATE, {}).values(), + lambda p: p.properties.policy_id, + ) + ] + ) + + for policy in delete_policies: + if existing_template := existing_template_map.get(policy): + existing_template.delete() + + generate_scp_resource_files_semaphore = NoqSemaphore( + generate_scp_resource_files, 25 + ) + + scp_policies: list[ + ServiceControlPolicyResourceFiles + ] = await generate_scp_resource_files_semaphore.process( + [ + { + "exe_message": exe_message, + "aws_account": aws_account_map[exe_message.provider_id], + "filtered_policies": set( + [ + dm.policy_id + for dm in detect_messages + if not dm.delete and dm.policy_id not in delete_policies + ] + ), + } + ] + ) + + else: + generate_scp_resource_files_semaphore = NoqSemaphore( + generate_scp_resource_files, 25 + ) + + scp_policies: list[ + ServiceControlPolicyResourceFiles + ] = await generate_scp_resource_files_semaphore.process( + [ + { + "exe_message": exe_message, + "aws_account": aws_account_map[exe_message.provider_id], # type: ignore + } + ] + ) + + log.info( + "Finished retrieving AWS SCP policy details", + accounts=list(aws_account_map.keys()), + ) + + with open( + exe_message.get_file_path( + *RESOURCE_DIR, + file_name_and_extension=f"output-{exe_message.provider_id}.json", + ), + "w", + ) as f: + f.write(json.dumps(scp_policies)) + + +async def generate_aws_scp_policy_templates( + exe_message: ExecutionMessage, + config: AWSConfig, + base_output_dir: str, + scp_template_map: dict, + detect_messages: Optional[list[Union[SCPPolicyMessageDetails, Any]]] = None, +): + """Generate AWS SCP policy templates + + Note: + this function is executed for each organization account (provider_id) + thus, each output file has one item in the list (scp_policies) + """ + + aws_account_map = await get_organizations_account_map(exe_message, config) + + existing_template_map = scp_template_map.get(AWS_SCP_POLICY_TEMPLATE, {}) + resource_dir = get_template_dir(base_output_dir) + + scp_policies: list[ + ServiceControlPolicyResourceFiles + ] = await exe_message.get_sub_exe_files( + *RESOURCE_DIR, + file_name_and_extension=f"output-{exe_message.provider_id}.json", + flatten_results=True, + ) # type: ignore + + policies: list[Union[ServiceControlPolicyCache, dict]] = [] + account_id: str + account_id, policies = scp_policies[0].values() # type: ignore + + tasks = [] + organization: AWSOrganization = list( + filter(lambda o: o.org_account_id == account_id, config.organizations) + )[0] + + for policy in policies: + tasks.append( + upsert_templated_scp_policies( + aws_account_map, + account_id, + policy, # type: ignore + resource_dir, + existing_template_map, + config, + organization, + ) + ) + + log.debug("Writing templated scp policies") + + templates: list[AwsScpPolicyTemplate] = await asyncio.gather(*tasks) + + # NEVER call this if messages are passed in because all_resource_ids will only contain those resources + if not detect_messages: + # if some templates are iambic managed, they will be None + all_resource_ids = set([t.identifier for t in templates if t is not None]) + delete_orphaned_templates( + list(existing_template_map.values()), all_resource_ids + ) + + log.info("Finished templated scp policies generation") + + return + + +async def get_organizations_account_map(exe_message, config): + aws_account_map = await get_aws_account_map(config) + + # SCP policies should be retrieved just for the organization account (provider_id) + if not exe_message.provider_id or not aws_account_map.get(exe_message.provider_id): + raise OrganizationAccountRequiredException() + + aws_account_map = { + exe_message.provider_id: aws_account_map[exe_message.provider_id] + } + + return aws_account_map + + +async def upsert_templated_scp_policies( + aws_account_map: dict, + account_id: str, + policy: ServiceControlPolicyCache, # type: ignore + resource_dir: str, + existing_template_map: dict, + config, + organization: AWSOrganization, +): + async with aiofiles.open(policy.get("file_path"), mode="r") as f: + content_dict = json.loads(await f.read()) + # policy = normalize_dict_keys(content_dict) # type: ignore + policy: ServiceControlPolicyItem = ServiceControlPolicyItem.parse_obj( + content_dict + ) + + file_path = get_template_file_path( + resource_dir, + policy.Name, + [aws_account_map[account_id].account_name], + aws_account_map, + ) + + template_params, template_properties = AwsScpPolicyTemplate.factory_template_props( + account_id, + policy, + config, + organization, + ) + + return create_or_update_template( + file_path, + existing_template_map, + policy.Name, + AwsScpPolicyTemplate, + template_params, + PolicyProperties(**template_properties), + list(aws_account_map.values()), + ) + + +def get_template_file_path( + managed_policy_dir: str, + policy_name: str, + included_accounts: list[str], + account_map: dict[str, AWSAccount], +): + if len(included_accounts) > 1: + separator = "multi_account" + elif included_accounts == ["*"] or included_accounts is None: + separator = "all_accounts" + else: + separator = included_accounts[0] + + file_name = ( + policy_name.replace(" ", "") + .replace("{{var.", "") + .replace("{{", "") + .replace("}}_", "_") + .replace("}}", "_") + .replace(".", "_") + .lower() + ) + return str(os.path.join(managed_policy_dir, separator, f"{file_name}.yaml")) diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py b/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py new file mode 100644 index 000000000..f2ed8bce6 --- /dev/null +++ b/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py @@ -0,0 +1,673 @@ +from __future__ import annotations + +import asyncio +import json +import random +from itertools import chain + +from deepdiff import DeepDiff +from git import TYPE_CHECKING +from tenacity import retry, stop_after_attempt, wait_exponential + +from iambic.core.context import ctx +from iambic.core.logger import log +from iambic.core.models import ProposedChange, ProposedChangeType +from iambic.core.utils import NoqSemaphore, aio_wrapper, plugin_apply_wrapper +from iambic.plugins.v0_1_0.aws.utils import boto_crud_call, legacy_paginated_search + +if TYPE_CHECKING: + from iambic.plugins.v0_1_0.aws.models import AWSAccount + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + ServiceControlPolicyItem, + ServiceControlPolicyTargetItem, + ) + + +async def list_policies( + client, filter="SERVICE_CONTROL_POLICY" +) -> list[ServiceControlPolicyItem]: + """Retrieves the list of all policies in an organization of a specified type.""" + + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + ServiceControlPolicyItem, + ) + + scp_policies = await legacy_paginated_search( + client.list_policies, + response_key="Policies", + **dict(Filter=filter), + ) + + scp_policies = [p for p in scp_policies if p["AwsManaged"] is False] + + list_targets_for_policy_semaphore = NoqSemaphore(list_targets_for_policy, 10) + describe_policy_semaphore = NoqSemaphore(get_policy_statements, 10) + list_tags_by_policy_semaphore = NoqSemaphore(list_tags_by_policy, 10) + + targets = await list_targets_for_policy_semaphore.process( + [{"client": client, "policyId": policy["Id"]} for policy in scp_policies] + ) + + statements = await describe_policy_semaphore.process( + [{"client": client, "policyId": policy["Id"]} for policy in scp_policies] + ) + + tags = await list_tags_by_policy_semaphore.process( + [{"client": client, "policyId": policy["Id"]} for policy in scp_policies] + ) + + return [ + ServiceControlPolicyItem.parse_obj( + { + **p, + "Targets": t, + "PolicyDocument": s, + "Tags": tg, + }, + ) + for p, t, s, tg in zip(scp_policies, targets, statements, tags) + ] + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +async def list_targets_for_policy( + client, policyId: str +) -> list[ServiceControlPolicyTargetItem]: + """ + Lists all the roots, organizational units (OUs), + and accounts that the specified policy is attached to. + """ + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + ServiceControlPolicyTargetItem, + ) + + targets = await legacy_paginated_search( + client.list_targets_for_policy, + response_key="Targets", + **dict(PolicyId=policyId), + ) + + return [ServiceControlPolicyTargetItem.parse_obj(t) for t in targets] + + +async def get_policy_statements(client, policyId: str): + policy = await describe_policy(client, policyId) + return policy.get("Content", {}) + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +async def describe_policy(client, policyId: str): + from iambic.plugins.v0_1_0.aws.organizations.scp.models import PolicyDocumentItem + + res = await boto_crud_call(client.describe_policy, PolicyId=policyId) + return dict( + PolicySummary=res.get("Policy", {}).get("PolicySummary", {}), + Content=PolicyDocumentItem.parse_obj( + json.loads(res.get("Policy", {}).get("Content", {})) + ), + ) + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +async def list_tags_by_policy(client, policyId: str) -> list[dict]: + """ + Lists tags that are attached to the specified resource. + """ + targets = await legacy_paginated_search( + client.list_tags_for_resource, + response_key="Tags", + **dict(ResourceId=policyId), + ) + + return targets + + +async def get_policy(client, policyId: str) -> ServiceControlPolicyItem: + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + ServiceControlPolicyItem, + ) + + policy = await describe_policy(client, policyId) + + list_tags_by_policy_semaphore = NoqSemaphore(list_tags_by_policy, 10) + list_targets_for_policy_semaphore = NoqSemaphore(list_targets_for_policy, 10) + + tags = await list_tags_by_policy_semaphore.process( + [{"client": client, "policyId": policyId}] + ) + + targets = await list_targets_for_policy_semaphore.process( + [{"client": client, "policyId": policyId}] + ) + + return ServiceControlPolicyItem.parse_obj( + { + **policy.get("PolicySummary", {}), # type: ignore + "Targets": targets[0], + "PolicyDocument": policy.get("Content", {}), + "Tags": tags[0], + }, + ) + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +async def detach_policy(client, policyId, targetId): + log.info(f"Detaching policy {policyId} from {targetId}") + await boto_crud_call( + client.detach_policy, + PolicyId=policyId, + TargetId=targetId, + ) + log.info(f"Detached policy {policyId} from {targetId}") + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=5, max=20), +) +async def delete_policy(client, policyId: str, *args, **kwargs): + """ + Deletes the specified policy from your organization. + Before you perform this operation, you must first detach + the policy from all organizational units (OUs), roots, and accounts. + """ + list_targets_for_policy_semaphore = NoqSemaphore(list_targets_for_policy, 10) + + targets = chain.from_iterable( + await list_targets_for_policy_semaphore.process( + [{"client": client, "policyId": policyId}] + ) + ) + + targets_tasks = [ + detach_policy(client, policyId, target.TargetId) for target in targets + ] + + await asyncio.gather(*targets_tasks) + + log.info(f"Deleting policy {policyId}") + + await boto_crud_call(client.delete_policy, PolicyId=policyId) + + log.info(f"Deleted policy {policyId}") + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +async def create_policy(client, policy): + if isinstance(policy["PolicyDocument"], dict): + policy["PolicyDocument"] = json.dumps(policy["PolicyDocument"]) + + res = await boto_crud_call( + client.create_policy, + Content=policy["PolicyDocument"], + Description=policy.get("Description", ""), + Name=policy.get("PolicyName", f"NewPolicy-{random.randint(0, 100):03d}"), + Type="SERVICE_CONTROL_POLICY", + ) + + policy.update( + PolicyId=res.get("Policy", {}).get("PolicySummary", {}).get("Id"), + PolicyName=res.get("Policy", {}).get("PolicySummary", {}).get("Name"), + ) + + return res.get("Policy", {}).get("PolicySummary", {}) + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=5, max=20), +) +async def describe_organization( + client, +): + """Retrieves information about the organization that the user's account belongs to. + This operation can be called from any account in the organization. + """ + res = await boto_crud_call(client.describe_organization) + return res.get("Organization", {}) + + +async def service_control_policy_is_enabled(client): + """Check if SCPs are enabled for the organization.""" + org = await describe_organization(client) + + return ( + len( + [ + apt + for apt in org.get("AvailablePolicyTypes", []) + if apt.get("Type") == "SERVICE_CONTROL_POLICY" + and apt.get("Status") == "ENABLED" + ] + ) + > 0 + ) + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +async def apply_update_policy( + client, + policy, + current_policy, + log_params, + *args, +) -> list[ProposedChange]: + """Apply update policy. + + Args: + client: organization client + policy: policy read from yaml + current_policy (dict): Response from boto3, based on ServiceControlPolicyItem. + log_params (dict): + + Returns: + list[ProposedChange]: list of proposed changes + """ + current_value = dict( + Name=current_policy.get("Name"), + Description=current_policy.get("Description"), + Content=current_policy["PolicyDocument"], + ) + + new_value = dict( + Name=policy.get("PolicyName"), + Description=policy.get("Description"), + Content=policy["PolicyDocument"], + ) + + diff = await aio_wrapper( + DeepDiff, + current_value, + new_value, + report_repetition=True, + ignore_order=True, + ) + + if not diff.get("values_changed"): + return [] + + tasks = [] + policy = policy.copy() + + if isinstance(policy["PolicyDocument"], dict): + policy["PolicyDocument"] = json.dumps(policy["PolicyDocument"]) + + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.UPDATE, + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + change_summary=diff.to_dict(), + current_value=current_value, + new_value=new_value, + ) # type: ignore + ] + + if ctx.execute: + apply_awaitable = boto_crud_call( + client.update_policy, + PolicyId=policy.get("PolicyId"), + Name=policy.get("PolicyName", f"NewPolicy-{random.randint(0, 100):03d}"), + Description=policy.get("Description", ""), + Content=policy["PolicyDocument"], + ) + + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) + + if tasks: + results: list[list[ProposedChange]] = await asyncio.gather(*tasks) + return list(chain.from_iterable(results)) + else: + return proposed_changes + + +async def apply_update_policy_targets( + client, + policy, + current_policy, + log_params, + aws_account: AWSAccount, + *args, +) -> list[ProposedChange]: + """Apply update policy targets. + + Args: + client: organization client + policy: policy read from yaml + current_policy (dict): Response from boto3, based on ServiceControlPolicyItem. + log_params (dict): + + Returns: + list[ProposedChange]: list of proposed changes + """ + tasks = [] + response = [] + + t, r = __remove_targets( + client, + policy, + current_policy, + log_params, + aws_account, + ) + + tasks += t + response += r + + t, r = __apply_targets( + client, + policy, + current_policy, + log_params, + aws_account, + ) + + tasks += t + response += r + + if tasks: + results: list[list[ProposedChange]] = await asyncio.gather(*tasks) + return list(chain.from_iterable(results)) + else: + return response + + +async def apply_update_policy_tags( + client, + policy, + current_policy, + log_params, + *args, +) -> list[ProposedChange]: + """Apply update policy tags. + + Args: + client: organization client + policy: policy read from yaml + current_policy (dict): Response from boto3, based on ServiceControlPolicyItem. + log_params (dict): + + Returns: + list[ProposedChange]: list of proposed changes + """ + + tasks = [] + response = [] + + t, r = __remove_tags( + client, + policy, + current_policy, + log_params, + ) + + tasks += t + response += r + + t, r = __apply_tags( + client, + policy, + current_policy, + log_params, + ) + + tasks += t + response += r + + if tasks: + results: list[list[ProposedChange]] = await asyncio.gather(*tasks) + return list(chain.from_iterable(results)) + else: + return response + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +def __apply_tags( + client, + policy, + current_policy, + log_params, +): + """Apply tags to policy""" + response = [] + tasks = [] + existing_tag_map = { + tag["Key"]: tag.get("Value") for tag in current_policy.get("Tags", []) + } + + tags_to_apply = [ + tag + for tag in policy.get("Tags", []) + if tag.get("Value") != existing_tag_map.get(tag["Key"]) + ] + + if tags_to_apply: + log_str = "New tags discovered in AWS." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.ATTACH, + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + attribute="tags", + current_value=[], + new_value=tags_to_apply, + ) # type: ignore + ] + response.extend(proposed_changes) + if ctx.execute: + log_str = f"{log_str} Adding tags..." + apply_awaitable = boto_crud_call( + client.tag_resource, + ResourceId=policy.get("PolicyId"), + Tags=tags_to_apply, + ) + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) + log.info(log_str, tags=tags_to_apply, **log_params) + + return tasks, response + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +def __remove_tags(client, policy, current_policy, log_params): + """Remove tags from the policy that are not in the template.""" + response = [] + tasks = [] + template_tag_map = {tag["Key"]: tag.get("Value") for tag in policy.get("Tags", [])} + + if tags_to_remove := [ + tag["Key"] + for tag in current_policy.get("Tags", []) + if not template_tag_map.get(tag["Key"]) + ]: + log_str = "Stale tags discovered." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.DETACH, + attribute="tags", + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + current_value=tags_to_remove, + new_value=[], + change_summary={"TagKeys": tags_to_remove}, + ) # type: ignore + ] + response.extend(proposed_changes) + + if ctx.execute: + log_str = f"{log_str} Removing tags..." + apply_awaitable = boto_crud_call( + client.untag_resource, + ResourceId=policy.get("PolicyId"), + TagKeys=tags_to_remove, + ) + tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) + log.info(log_str, tags=tags_to_remove, **log_params) + + return tasks, response + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +def __apply_targets( + client, + policy, + current_policy, + log_params, + aws_account: AWSAccount, +): + """Apply targets to policy.""" + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + PolicyTargetProperties, + ) + + response = [] + tasks = [] + + targets = list( + chain.from_iterable( + policy.get( + "Targets", dict(OrganizationalUnits=[], Accounts=[], Roots=[]) + ).values() + ) + ) + targets = PolicyTargetProperties.unparse_targets(targets, aws_account.aws_config) + + current_targets = list( + map(lambda t: t.get("TargetId"), current_policy.get("Targets")) + ) + + if targets_to_apply := [tag for tag in targets if tag not in current_targets]: + log_str = "New targets discovered." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.ATTACH, + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + attribute="targets", + current_value=current_targets, + new_value=current_targets + targets_to_apply, + ) # type: ignore + ] + response.extend(proposed_changes) + if ctx.execute: + log_str = f"{log_str} Adding targets..." + + async def attach_policies(): + """Due to ConcurrentModificationException, we need to execute it one at a time.""" + for target in targets_to_apply: + await boto_crud_call( + client.attach_policy, + PolicyId=policy.get("PolicyId"), + TargetId=target, + ) + + tasks.append(plugin_apply_wrapper(attach_policies(), proposed_changes)) + + log.info(log_str, tags=targets_to_apply, **log_params) + + return tasks, response + + +@retry( + reraise=True, + stop=stop_after_attempt(6), + wait=wait_exponential(multiplier=1, min=4, max=15), +) +def __remove_targets( + client, + policy, + current_policy, + log_params, + aws_account: AWSAccount, +): + """Remove targets from policy that are not in the template.""" + from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + PolicyTargetProperties, + ) + + response = [] + tasks = [] + + targets = list( + chain.from_iterable( + policy.get( + "Targets", dict(OrganizationalUnits=[], Accounts=[], Roots=[]) + ).values() + ) + ) + targets = PolicyTargetProperties.unparse_targets(targets, aws_account.aws_config) + + current_targets = list( + map(lambda t: t.get("TargetId"), current_policy.get("Targets")) + ) + + if targets_to_remove := [ + target for target in current_targets if target not in targets + ]: + log_str = "Stale targets discovered." + proposed_changes = [ + ProposedChange( + change_type=ProposedChangeType.DETACH, + attribute="targets", + resource_type=log_params.get("resource_type"), + resource_id=policy.get("Name"), + current_value=targets_to_remove, + new_value=[], + change_summary={"Targets": targets_to_remove}, + ) # type: ignore + ] + response.extend(proposed_changes) + + if ctx.execute: + log_str = f"{log_str} Removing targets..." + + async def detach_policies(): + """Due to ConcurrentModificationException, we need to execute it one at a time.""" + for target in targets_to_remove: + await boto_crud_call( + client.detach_policy, + PolicyId=policy.get("PolicyId"), + TargetId=target, + ) + + tasks.append(plugin_apply_wrapper(detach_policies(), proposed_changes)) + log.info(log_str, tags=targets_to_remove, **log_params) + + return tasks, response diff --git a/iambic/plugins/v0_1_0/aws/utils.py b/iambic/plugins/v0_1_0/aws/utils.py index 7fa7c0a9d..31ac83516 100644 --- a/iambic/plugins/v0_1_0/aws/utils.py +++ b/iambic/plugins/v0_1_0/aws/utils.py @@ -106,7 +106,9 @@ async def paginated_search( for response_key in response_keys: results[response_key].extend(response.get(response_key, [])) - if not response["IsTruncated"] or (max_results and len(results) >= max_results): + if not response.get("IsTruncated") or ( + max_results and len(results) >= max_results + ): return results if retain_key else results[response_key] else: search_kwargs["Marker"] = response["Marker"] diff --git a/poetry.lock b/poetry.lock index 4a838d916..d5d92bfd6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.0 and should not be changed by hand. [[package]] name = "aenum" version = "3.1.12" description = "Advanced Enumerations (compatible with Python's stdlib Enum), NamedTuples, and NamedConstants" -category = "main" optional = false python-versions = "*" files = [ @@ -17,7 +16,6 @@ files = [ name = "aiofiles" version = "23.1.0" description = "File support for asyncio." -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -29,7 +27,6 @@ files = [ name = "aiohttp" version = "3.8.4" description = "Async http client/server framework (asyncio)" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -138,7 +135,6 @@ speedups = ["Brotli", "aiodns", "cchardet"] name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -153,7 +149,6 @@ frozenlist = ">=1.1.0" name = "asgiref" version = "3.6.0" description = "ASGI specs, helper code, and adapters" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -168,7 +163,6 @@ tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] name = "async-timeout" version = "4.0.2" description = "Timeout context manager for asyncio programs" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -180,7 +174,6 @@ files = [ name = "asyncache" version = "0.3.1" description = "Helpers to use cachetools with async code." -category = "main" optional = false python-versions = ">=3.8,<4.0" files = [ @@ -195,7 +188,6 @@ cachetools = ">=5.2.0,<6.0.0" name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -214,7 +206,6 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "aws-error-utils" version = "2.7.0" description = "Error-handling functions for boto3/botocore" -category = "main" optional = false python-versions = ">=3.7,<4" files = [ @@ -229,7 +220,6 @@ botocore = "*" name = "aws-sam-translator" version = "1.66.0" description = "AWS SAM Translator is a library that transform SAM templates into AWS CloudFormation templates" -category = "dev" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ @@ -238,19 +228,18 @@ files = [ ] [package.dependencies] -boto3 = ">=1.19.5,<2.0.0" +boto3 = ">=1.19.5,<2.dev0" jsonschema = ">=3.2,<5" pydantic = ">=1.8,<2.0" typing-extensions = ">=4.4,<5" [package.extras] -dev = ["black (==23.1.0)", "boto3 (>=1.23,<2)", "boto3-stubs[appconfig,serverlessrepo] (>=1.19.5,<2.0.0)", "coverage (>=5.3,<8)", "dateparser (>=1.1,<2.0)", "mypy (>=1.1.0,<1.2.0)", "parameterized (>=0.7,<1.0)", "pytest (>=6.2,<8)", "pytest-cov (>=2.10,<5)", "pytest-env (>=0.6,<1)", "pytest-rerunfailures (>=9.1,<12)", "pytest-xdist (>=2.5,<4)", "pyyaml (>=6.0,<7.0)", "requests (>=2.28,<3.0)", "ruamel.yaml (==0.17.21)", "ruff (==0.0.261)", "tenacity (>=8.0,<9.0)", "types-PyYAML (>=6.0,<7.0)", "types-jsonschema (>=3.2,<4.0)"] +dev = ["black (==23.1.0)", "boto3 (>=1.23,<2)", "boto3-stubs[appconfig,serverlessrepo] (>=1.19.5,<2.dev0)", "coverage (>=5.3,<8)", "dateparser (>=1.1,<2.0)", "mypy (>=1.1.0,<1.2.0)", "parameterized (>=0.7,<1.0)", "pytest (>=6.2,<8)", "pytest-cov (>=2.10,<5)", "pytest-env (>=0.6,<1)", "pytest-rerunfailures (>=9.1,<12)", "pytest-xdist (>=2.5,<4)", "pyyaml (>=6.0,<7.0)", "requests (>=2.28,<3.0)", "ruamel.yaml (==0.17.21)", "ruff (==0.0.261)", "tenacity (>=8.0,<9.0)", "types-PyYAML (>=6.0,<7.0)", "types-jsonschema (>=3.2,<4.0)"] [[package]] name = "aws-xray-sdk" version = "2.12.0" description = "The AWS X-Ray SDK for Python (the SDK) enables Python developers to record and emit information from within their applications to the AWS X-Ray service." -category = "dev" optional = false python-versions = "*" files = [ @@ -266,7 +255,6 @@ wrapt = "*" name = "black" version = "23.3.0" description = "The uncompromising code formatter." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -316,7 +304,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "boto3" version = "1.26.133" description = "The AWS SDK for Python" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -336,7 +323,6 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.29.133" description = "Low-level, data-driven core of boto 3." -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -356,7 +342,6 @@ crt = ["awscrt (==0.16.9)"] name = "cachetools" version = "5.3.0" description = "Extensible memoizing collections and decorators" -category = "main" optional = false python-versions = "~=3.7" files = [ @@ -368,7 +353,6 @@ files = [ name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -380,7 +364,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = "*" files = [ @@ -457,7 +440,6 @@ pycparser = "*" name = "cfgv" version = "3.3.1" description = "Validate configuration and produce human readable error messages." -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -469,7 +451,6 @@ files = [ name = "cfn-lint" version = "0.77.5" description = "Checks CloudFormation templates for practices and behaviour that could potentially be improved" -category = "dev" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ @@ -493,7 +474,6 @@ sympy = ">=1.0.0" name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -578,7 +558,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -593,7 +572,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -605,7 +583,6 @@ files = [ name = "coverage" version = "7.2.5" description = "Code coverage measurement for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -672,7 +649,6 @@ toml = ["tomli"] name = "cryptography" version = "39.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -718,7 +694,6 @@ tox = ["tox"] name = "dateparser" version = "1.1.8" description = "Date parsing library designed to parse dates from HTML pages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -741,7 +716,6 @@ langdetect = ["langdetect"] name = "deepdiff" version = "6.3.0" description = "Deep Difference and Search of any Python object/data. Recreate objects by adding adding deltas to each other." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -760,7 +734,6 @@ optimize = ["orjson"] name = "deprecated" version = "1.2.13" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -778,7 +751,6 @@ dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version name = "dictdiffer" version = "0.9.0" description = "Dictdiffer is a library that helps you to diff and patch dictionaries." -category = "main" optional = false python-versions = "*" files = [ @@ -796,7 +768,6 @@ tests = ["check-manifest (>=0.42)", "mock (>=1.3.0)", "pytest (==5.4.3)", "pytes name = "distlib" version = "0.3.6" description = "Distribution utilities" -category = "main" optional = false python-versions = "*" files = [ @@ -808,7 +779,6 @@ files = [ name = "docker" version = "6.1.2" description = "A Python library for the Docker Engine API." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -830,7 +800,6 @@ ssh = ["paramiko (>=2.4.3)"] name = "ecdsa" version = "0.18.0" description = "ECDSA cryptographic signature library (pure python)" -category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -849,7 +818,6 @@ gmpy2 = ["gmpy2"] name = "exceptiongroup" version = "1.1.1" description = "Backport of PEP 654 (exception groups)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -864,7 +832,6 @@ test = ["pytest (>=6)"] name = "execnet" version = "1.9.0" description = "execnet: rapid multi-Python deployment" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -879,7 +846,6 @@ testing = ["pre-commit"] name = "faker" version = "18.7.0" description = "Faker is a Python package that generates fake data for you." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -894,7 +860,6 @@ python-dateutil = ">=2.4" name = "filelock" version = "3.12.0" description = "A platform independent file lock." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -910,7 +875,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "p name = "flake8" version = "6.0.0" description = "the modular source code checker: pep8 pyflakes and co" -category = "main" optional = false python-versions = ">=3.8.1" files = [ @@ -927,7 +891,6 @@ pyflakes = ">=3.0.0,<3.1.0" name = "flatdict" version = "4.0.1" description = "Python module for interacting with nested dicts as a single level dict with delimited keys." -category = "main" optional = false python-versions = "*" files = [ @@ -938,7 +901,6 @@ files = [ name = "frozenlist" version = "1.3.3" description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1022,7 +984,6 @@ files = [ name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1037,7 +998,6 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.31" description = "GitPython is a Python library used to interact with Git repositories" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1052,7 +1012,6 @@ gitdb = ">=4.0.1,<5" name = "google-api-core" version = "2.11.0" description = "Google API client core library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1075,7 +1034,6 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] name = "google-api-python-client" version = "2.86.0" description = "Google API Client Library for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1084,7 +1042,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.19.0,<3.0.0dev" google-auth-httplib2 = ">=0.1.0" httplib2 = ">=0.15.0,<1dev" @@ -1094,7 +1052,6 @@ uritemplate = ">=3.0.1,<5" name = "google-auth" version = "2.18.0" description = "Google Authentication Library" -category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" files = [ @@ -1120,7 +1077,6 @@ requests = ["requests (>=2.20.0,<3.0.0dev)"] name = "google-auth-httplib2" version = "0.1.0" description = "Google Authentication Library: httplib2 transport" -category = "main" optional = false python-versions = "*" files = [ @@ -1137,7 +1093,6 @@ six = "*" name = "googleapis-common-protos" version = "1.59.0" description = "Common protobufs used in Google APIs" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1155,7 +1110,6 @@ grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] name = "graphql-core" version = "3.2.3" description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." -category = "dev" optional = false python-versions = ">=3.6,<4" files = [ @@ -1167,7 +1121,6 @@ files = [ name = "httplib2" version = "0.22.0" description = "A comprehensive HTTP client library." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1182,7 +1135,6 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 name = "identify" version = "2.5.24" description = "File identification library for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1197,7 +1149,6 @@ license = ["ukkonen"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1209,7 +1160,6 @@ files = [ name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1221,7 +1171,6 @@ files = [ name = "isort" version = "5.12.0" description = "A Python utility / library to sort Python imports." -category = "main" optional = false python-versions = ">=3.8.0" files = [ @@ -1239,7 +1188,6 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1257,7 +1205,6 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1269,7 +1216,6 @@ files = [ name = "jschema-to-python" version = "1.2.3" description = "Generate source code for Python classes from a JSON schema." -category = "dev" optional = false python-versions = ">= 2.7" files = [ @@ -1286,7 +1232,6 @@ pbr = "*" name = "jsondiff" version = "2.0.0" description = "Diff JSON and JSON-like structures in Python" -category = "dev" optional = false python-versions = "*" files = [ @@ -1298,7 +1243,6 @@ files = [ name = "jsonpatch" version = "1.32" description = "Apply JSON-Patches (RFC 6902)" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1313,7 +1257,6 @@ jsonpointer = ">=1.9" name = "jsonpickle" version = "3.0.1" description = "Python library for serializing any arbitrary object graph into JSON" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1330,7 +1273,6 @@ testing-libs = ["simplejson", "ujson"] name = "jsonpointer" version = "2.3" description = "Identify specific nodes in a JSON document (RFC 6901)" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1342,7 +1284,6 @@ files = [ name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1362,7 +1303,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jsonschema-spec" version = "0.1.4" description = "JSONSchema Spec with object-oriented paths" -category = "dev" optional = false python-versions = ">=3.7.0,<4.0.0" files = [ @@ -1380,7 +1320,6 @@ typing-extensions = ">=4.3.0,<5.0.0" name = "jsonschema2md2" version = "0.6.0" description = "Convert JSON Schema to human-readable Markdown documentation" -category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -1396,7 +1335,6 @@ PyYAML = ">=5.1" name = "junit-xml" version = "1.9" description = "Creates JUnit XML test result documents that can be read by tools such as Jenkins" -category = "dev" optional = false python-versions = "*" files = [ @@ -1411,7 +1349,6 @@ six = "*" name = "lazy-object-proxy" version = "1.9.0" description = "A fast and thorough lazy object proxy." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1457,7 +1394,6 @@ files = [ name = "markdown-it-py" version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1482,7 +1418,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1542,7 +1477,6 @@ files = [ name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1554,7 +1488,6 @@ files = [ name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1566,7 +1499,6 @@ files = [ name = "mock" version = "5.0.2" description = "Rolling backport of unittest.mock for all Pythons" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1583,7 +1515,6 @@ test = ["pytest", "pytest-cov"] name = "mock-generator" version = "2.4.0" description = "Generate python mocks and assertions quickly" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1599,7 +1530,6 @@ pyperclip = "1.8.2" name = "moto" version = "4.1.9" description = "" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1659,7 +1589,6 @@ xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" -category = "dev" optional = false python-versions = "*" files = [ @@ -1677,7 +1606,6 @@ tests = ["pytest (>=4.6)"] name = "msal" version = "1.22.0" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." -category = "main" optional = false python-versions = "*" files = [ @@ -1697,7 +1625,6 @@ broker = ["pymsalruntime (>=0.13.2,<0.14)"] name = "multidict" version = "6.0.4" description = "multidict implementation" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1781,7 +1708,6 @@ files = [ name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1793,7 +1719,6 @@ files = [ name = "networkx" version = "3.1" description = "Python package for creating and manipulating graphs and networks" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1812,7 +1737,6 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] name = "nodeenv" version = "1.8.0" description = "Node.js virtual environment builder" -category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -1827,7 +1751,6 @@ setuptools = "*" name = "okta" version = "2.9.2" description = "Python SDK for the Okta Management API" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1850,7 +1773,6 @@ yarl = "*" name = "openapi-schema-validator" version = "0.4.4" description = "OpenAPI schema validation for Python" -category = "dev" optional = false python-versions = ">=3.7.0,<4.0.0" files = [ @@ -1869,7 +1791,6 @@ docs = ["sphinx (>=5.3.0,<6.0.0)", "sphinx-immaterial (>=0.11.0,<0.12.0)"] name = "openapi-spec-validator" version = "0.5.6" description = "OpenAPI 2.0 (aka Swagger) and OpenAPI 3 spec validator" -category = "dev" optional = false python-versions = ">=3.7.0,<4.0.0" files = [ @@ -1891,7 +1812,6 @@ requests = ["requests"] name = "ordered-set" version = "4.1.0" description = "An OrderedSet is a custom MutableSet that remembers its order, so that every" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1906,7 +1826,6 @@ dev = ["black", "mypy", "pytest"] name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1918,7 +1837,6 @@ files = [ name = "pathable" version = "0.4.3" description = "Object-oriented paths" -category = "dev" optional = false python-versions = ">=3.7.0,<4.0.0" files = [ @@ -1930,7 +1848,6 @@ files = [ name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1942,7 +1859,6 @@ files = [ name = "pbr" version = "5.11.1" description = "Python Build Reasonableness" -category = "dev" optional = false python-versions = ">=2.6" files = [ @@ -1954,7 +1870,6 @@ files = [ name = "platformdirs" version = "3.5.1" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1970,7 +1885,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1986,7 +1900,6 @@ testing = ["pytest", "pytest-benchmark"] name = "pre-commit" version = "3.3.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2005,7 +1918,6 @@ virtualenv = ">=20.10.0" name = "prompt-toolkit" version = "3.0.38" description = "Library for building powerful interactive command lines in Python" -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -2020,7 +1932,6 @@ wcwidth = "*" name = "protobuf" version = "4.23.0" description = "" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2043,7 +1954,6 @@ files = [ name = "py-partiql-parser" version = "0.3.0" description = "Pure Python PartiQL Parser" -category = "dev" optional = false python-versions = "*" files = [ @@ -2058,7 +1968,6 @@ dev = ["black (==22.6.0)", "flake8", "mypy (==0.971)", "pytest", "sure (==2.0.0) name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -2070,7 +1979,6 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -2085,7 +1993,6 @@ pyasn1 = ">=0.4.6,<0.6.0" name = "pycodestyle" version = "2.10.0" description = "Python style guide checker" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2097,7 +2004,6 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2109,7 +2015,6 @@ files = [ name = "pycryptodome" version = "3.17" description = "Cryptographic library for Python" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2152,7 +2057,6 @@ files = [ name = "pycryptodomex" version = "3.17" description = "Cryptographic library for Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2195,7 +2099,6 @@ files = [ name = "pydantic" version = "1.10.7" description = "Data validation and settings management using python type hints" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2248,7 +2151,6 @@ email = ["email-validator (>=1.0.3)"] name = "pydantic-factories" version = "1.17.3" description = "Mock data generation for pydantic based models and python dataclasses" -category = "main" optional = false python-versions = ">=3.8,<4.0" files = [ @@ -2265,7 +2167,6 @@ typing-extensions = "*" name = "pydash" version = "7.0.3" description = "The kitchen sink of Python utility libraries for doing \"stuff\" in a functional way. Based on the Lo-Dash Javascript library." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2283,7 +2184,6 @@ dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8 name = "pyflakes" version = "3.0.1" description = "passive checker of Python programs" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2295,7 +2195,6 @@ files = [ name = "pygithub" version = "1.57" description = "Use the full Github API v3" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2316,7 +2215,6 @@ integrations = ["cryptography"] name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2331,7 +2229,6 @@ plugins = ["importlib-metadata"] name = "pyjwt" version = "2.7.0" description = "JSON Web Token implementation in Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2352,7 +2249,6 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pynacl" version = "1.5.0" description = "Python binding to the Networking and Cryptography (NaCl) library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2379,7 +2275,6 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] name = "pyopenssl" version = "23.1.1" description = "Python wrapper module around the OpenSSL library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2398,7 +2293,6 @@ test = ["flaky", "pretend", "pytest (>=3.0.1)"] name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -2413,7 +2307,6 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyperclip" version = "1.8.2" description = "A cross-platform clipboard module for Python. (Only handles plain text for now.)" -category = "dev" optional = false python-versions = "*" files = [ @@ -2424,7 +2317,6 @@ files = [ name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2461,7 +2353,6 @@ files = [ name = "pytest" version = "7.3.1" description = "pytest: simple powerful testing with Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2484,7 +2375,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-asyncio" version = "0.21.0" description = "Pytest support for asyncio" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2503,7 +2393,6 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "pytest-cov" version = "4.0.0" description = "Pytest plugin for measuring coverage." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2522,7 +2411,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-mock" version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2540,7 +2428,6 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] name = "pytest-mock-generator" version = "1.2.0" description = "A pytest fixture wrapper for https://pypi.org/project/mock-generator" -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -2555,7 +2442,6 @@ mock-generator = ">=2.4,<3.0" name = "pytest-rerunfailures" version = "11.1.2" description = "pytest plugin to re-run tests to eliminate flaky failures" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2571,7 +2457,6 @@ pytest = ">=5.3" name = "pytest-xdist" version = "3.3.0" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2592,7 +2477,6 @@ testing = ["filelock"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -2607,7 +2491,6 @@ six = ">=1.5" name = "python-jose" version = "3.3.0" description = "JOSE implementation in Python" -category = "main" optional = false python-versions = "*" files = [ @@ -2630,7 +2513,6 @@ pycryptodome = ["pyasn1", "pycryptodome (>=3.3.1,<4.0.0)"] name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" files = [ @@ -2642,7 +2524,6 @@ files = [ name = "pywin32" version = "306" description = "Python for Window Extensions" -category = "dev" optional = false python-versions = "*" files = [ @@ -2666,7 +2547,6 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2716,7 +2596,6 @@ files = [ name = "questionary" version = "1.10.0" description = "Python library to build pretty command line user prompts ⭐️" -category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -2734,7 +2613,6 @@ docs = ["Sphinx (>=3.3,<4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphin name = "regex" version = "2023.5.5" description = "Alternative regular expression module, to replace re." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2832,7 +2710,6 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2854,7 +2731,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "responses" version = "0.23.1" description = "A utility library for mocking out the `requests` Python library." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2875,7 +2751,6 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "rfc3339-validator" version = "0.1.4" description = "A pure python RFC3339 validator" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2890,7 +2765,6 @@ six = "*" name = "rich" version = "13.3.5" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -2909,7 +2783,6 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -category = "main" optional = false python-versions = ">=3.6,<4" files = [ @@ -2924,7 +2797,6 @@ pyasn1 = ">=0.1.3" name = "ruamel-yaml" version = "0.17.26" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -category = "main" optional = false python-versions = ">=3" files = [ @@ -2943,7 +2815,6 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] name = "ruamel-yaml-clib" version = "0.2.7" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -2989,7 +2860,6 @@ files = [ name = "s3transfer" version = "0.6.1" description = "An Amazon S3 Transfer Manager" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -3007,7 +2877,6 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "sarif-om" version = "1.0.4" description = "Classes implementing the SARIF 2.1.0 object model." -category = "dev" optional = false python-versions = ">= 2.7" files = [ @@ -3023,7 +2892,6 @@ pbr = "*" name = "setuptools" version = "67.7.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3040,7 +2908,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -3052,7 +2919,6 @@ files = [ name = "slack-bolt" version = "1.18.0" description = "The Bolt Framework for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3074,7 +2940,6 @@ testing-without-asyncio = ["Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "Werk name = "slack-sdk" version = "3.21.3" description = "The Slack API Platform SDK for Python" -category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -3090,7 +2955,6 @@ testing = ["Flask (>=1,<2)", "Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "We name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3102,7 +2966,6 @@ files = [ name = "sshpubkeys" version = "3.3.1" description = "SSH public key parser" -category = "dev" optional = false python-versions = ">=3" files = [ @@ -3121,7 +2984,6 @@ dev = ["twine", "wheel", "yapf"] name = "stringcase" version = "1.2.0" description = "String case converter." -category = "main" optional = false python-versions = "*" files = [ @@ -3132,7 +2994,6 @@ files = [ name = "structlog" version = "22.3.0" description = "Structured Logging for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3150,7 +3011,6 @@ typing = ["mypy", "rich", "twisted"] name = "sympy" version = "1.12" description = "Computer algebra system (CAS) in Python" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3161,11 +3021,24 @@ files = [ [package.dependencies] mpmath = ">=0.19" +[[package]] +name = "tenacity" +version = "8.2.2" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.6" +files = [ + {file = "tenacity-8.2.2-py3-none-any.whl", hash = "sha256:2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0"}, + {file = "tenacity-8.2.2.tar.gz", hash = "sha256:43af037822bd0029025877f3b2d97cc4d7bb0c2991000a3d59d71517c5c969e0"}, +] + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + [[package]] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3177,7 +3050,6 @@ files = [ name = "tomlkit" version = "0.11.8" description = "Style preserving TOML library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3189,7 +3061,6 @@ files = [ name = "types-aiofiles" version = "23.1.0.2" description = "Typing stubs for aiofiles" -category = "dev" optional = false python-versions = "*" files = [ @@ -3201,7 +3072,6 @@ files = [ name = "types-cachetools" version = "5.3.0.5" description = "Typing stubs for cachetools" -category = "dev" optional = false python-versions = "*" files = [ @@ -3213,7 +3083,6 @@ files = [ name = "types-dateparser" version = "1.1.4.9" description = "Typing stubs for dateparser" -category = "main" optional = false python-versions = "*" files = [ @@ -3225,7 +3094,6 @@ files = [ name = "types-mock" version = "5.0.0.6" description = "Typing stubs for mock" -category = "main" optional = false python-versions = "*" files = [ @@ -3237,7 +3105,6 @@ files = [ name = "types-pyyaml" version = "6.0.12.9" description = "Typing stubs for PyYAML" -category = "dev" optional = false python-versions = "*" files = [ @@ -3249,7 +3116,6 @@ files = [ name = "types-ujson" version = "5.7.0.5" description = "Typing stubs for ujson" -category = "dev" optional = false python-versions = "*" files = [ @@ -3261,7 +3127,6 @@ files = [ name = "typing-extensions" version = "4.6.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3273,7 +3138,6 @@ files = [ name = "tzdata" version = "2023.3" description = "Provider of IANA time zone data" -category = "main" optional = false python-versions = ">=2" files = [ @@ -3285,7 +3149,6 @@ files = [ name = "tzlocal" version = "5.0" description = "tzinfo object for the local timezone" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3303,7 +3166,6 @@ devenv = ["black", "check-manifest", "flake8", "pyroma", "pytest (>=4.3)", "pyte name = "ujson" version = "5.7.0" description = "Ultra fast JSON encoder and decoder for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3378,7 +3240,6 @@ files = [ name = "uritemplate" version = "4.1.1" description = "Implementation of RFC 6570 URI Templates" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3390,7 +3251,6 @@ files = [ name = "urllib3" version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -3407,7 +3267,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "virtualenv" version = "20.23.0" description = "Virtual Python Environment builder" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3428,7 +3287,6 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "coverage-enable-subprocess name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" -category = "main" optional = false python-versions = "*" files = [ @@ -3440,7 +3298,6 @@ files = [ name = "websocket-client" version = "1.5.1" description = "WebSocket client for Python with low level API options" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3457,7 +3314,6 @@ test = ["websockets"] name = "werkzeug" version = "2.3.4" description = "The comprehensive WSGI web application library." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3475,7 +3331,6 @@ watchdog = ["watchdog (>=2.3)"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -3560,7 +3415,6 @@ files = [ name = "xmltodict" version = "0.13.0" description = "Makes working with XML feel like you are working with JSON" -category = "main" optional = false python-versions = ">=3.4" files = [ @@ -3572,7 +3426,6 @@ files = [ name = "xxhash" version = "3.2.0" description = "Python binding for xxHash" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3680,7 +3533,6 @@ files = [ name = "yarl" version = "1.9.2" description = "Yet another URL library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3767,4 +3619,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "27ebee7daf55a1bd407c867bd29381cae76bc59eb5b894fa35ab1767bb7be6f3" +content-hash = "549cf340dcb97e0d4e78b1bb48d00c5003a1ad83d7d6840afe67682871e7f140" diff --git a/pyproject.toml b/pyproject.toml index 2a2045ffd..bb8e14574 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ pyopenssl = "^23.0.0" stringcase = "^1.2.0" tomlkit = "^0.11.8" typing-extensions = "^4.6.1" +tenacity = "^8.2.2" [tool.poetry.scripts] iambic = "iambic.main:cli" diff --git a/test/core/test_utils.py b/test/core/test_utils.py index 1cda89018..90ceff88b 100644 --- a/test/core/test_utils.py +++ b/test/core/test_utils.py @@ -15,6 +15,7 @@ GlobalRetryController, convert_between_json_and_yaml, create_commented_map, + evaluate_on_provider, normalize_dict_keys, simplify_dt, sort_dict, @@ -291,3 +292,13 @@ def test_convert_between_json_and_yaml(): json_output = convert_between_json_and_yaml(yaml_input) expected_json_output = '{\n "MyKey": {\n "InnerKey": "value"\n }\n}' assert json_output == expected_json_output + + +def test_evaluate_on_provider_organization_account(mocker): + resource = mocker.Mock() + provider_details = mocker.Mock() + + resource.organization_account_needed = True + provider_details.organization_account = True + + assert evaluate_on_provider(resource, provider_details) diff --git a/test/plugins/v0_1_0/aws/iam/role/test_models.py b/test/plugins/v0_1_0/aws/iam/role/test_models.py index d41193bb6..241aa0299 100644 --- a/test/plugins/v0_1_0/aws/iam/role/test_models.py +++ b/test/plugins/v0_1_0/aws/iam/role/test_models.py @@ -1,5 +1,7 @@ from __future__ import annotations +import re + import pytest from iambic.core.template_generation import merge_model @@ -67,7 +69,9 @@ def test_merge_role_template_from_base_model_to_unsupported_type( assert type(new_document.properties.permissions_boundary) is float with pytest.raises( TypeError, - match=f"Type of {type(new_document.properties.permissions_boundary)} is not supported. {IAMBIC_ERR_MSG}", + match=re.escape( + f"Type of {type(new_document.properties.permissions_boundary)}(permissions_boundary) is not supported. {IAMBIC_ERR_MSG}" + ), ): _ = merge_model(new_document, existing_document, aws_accounts) diff --git a/test/plugins/v0_1_0/aws/organizations/__init__.py b/test/plugins/v0_1_0/aws/organizations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/plugins/v0_1_0/aws/organizations/conftest.py b/test/plugins/v0_1_0/aws/organizations/conftest.py new file mode 100644 index 000000000..55026d065 --- /dev/null +++ b/test/plugins/v0_1_0/aws/organizations/conftest.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import os +import shutil +import tempfile +from test.plugins.v0_1_0.aws.iam.policy.test_utils import ( # noqa: F401 # intentional for mocks + EXAMPLE_POLICY_DOCUMENT, +) +from test.plugins.v0_1_0.aws.organizations.scp.test_utils import ( + EXAMPLE_ACCOUNT_EMAIL, + EXAMPLE_ACCOUNT_NAME, + EXAMPLE_ORGANIZATIONAL_UNIT_NAME, + EXAMPLE_POLICY_DESCRIPTION, + EXAMPLE_POLICY_NAME, +) + +import boto3 +import pytest +from moto import mock_organizations, mock_sts +from moto.organizations.models import FakePolicy + +import iambic +from iambic.core.iambic_enum import Command +from iambic.core.models import ExecutionMessage +from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig +from iambic.plugins.v0_1_0.aws.models import AWSAccount, AWSOrganization + +TEST_TEMPLATE_DIR = "resources/aws/organizations/scp" +TEST_TEMPLATE_PATH = f"{TEST_TEMPLATE_DIR}/example_policy.yaml" + + +@pytest.fixture +def mock_fs(): + temp_templates_directory = tempfile.mkdtemp( + prefix="iambic_test_temp_templates_directory" + ) + + try: + os.makedirs(f"{temp_templates_directory}/{TEST_TEMPLATE_DIR}") + + test_template_path = f"{temp_templates_directory}/{TEST_TEMPLATE_PATH}" + with open(test_template_path, "w") as f: + f.write("") + + setattr(iambic.core.utils, "__WRITABLE_DIRECTORY__", temp_templates_directory) + + yield test_template_path, temp_templates_directory + finally: + try: + shutil.rmtree(temp_templates_directory) + except Exception as e: + print(e) + + +@pytest.fixture +def mock_execution_message(): + def factory(): + message = ExecutionMessage( + execution_id="fake_execution_id", + command=Command.IMPORT, + ) # type: ignore + return message + + return factory + + +@pytest.fixture +def mock_aws_account(): + account = AWSAccount( + account_id="123456789012", + account_name="example_account", + hub_role_arn="arn:aws:iam::123456789012:role/example-hub-role", + spoke_role_arn="arn:aws:iam::123456789012:role/example-spoke-role", + ) # type: ignore + + return account + + +@pytest.fixture +def mock_aws_organization(): + account = AWSOrganization( + org_name="main-organization", + org_id="123456789012"[::-1], + org_account_id="123456789012", + hub_role_arn="arn:aws:iam::123456789012:role/example-hub-role", + ) # type: ignore + return account # type: ignore + + +@pytest.fixture +def mock_aws_config(mock_aws_account, mock_aws_organization): + def factory(): + account = mock_aws_account + org = mock_aws_organization + + config = AWSConfig( + accounts=[mock_aws_account], + organizations=[mock_aws_organization], + ) # type: ignore + + account.set_account_organization_details(org, config) + + return config + + return factory + + +@pytest.fixture +def mock_organizations_client(): + with mock_organizations(), mock_sts(): + client = boto3.client("organizations") + org = client.create_organization(FeatureSet="ALL")["Organization"] + root = client.list_roots()["Roots"][0] + account = client.create_account( + AccountName=EXAMPLE_ACCOUNT_NAME, + Email=EXAMPLE_ACCOUNT_EMAIL, + )["CreateAccountStatus"] + org_unit = client.create_organizational_unit( + ParentId=root["Id"], Name=EXAMPLE_ORGANIZATIONAL_UNIT_NAME + )["OrganizationalUnit"] + + _current_init = FakePolicy.__init__ + + def new_init(self, organization, **kwargs): + # CHECK pull request https://github.com/getmoto/moto/pull/6338 + self.tags = kwargs.get("Tags", {}) + _current_init(self, organization, **kwargs) + + FakePolicy.__init__ = new_init + + policy = client.create_policy( + Content=EXAMPLE_POLICY_DOCUMENT, + Description=EXAMPLE_POLICY_DESCRIPTION, + Name=EXAMPLE_POLICY_NAME, + Type="SERVICE_CONTROL_POLICY", + )["Policy"]["PolicySummary"] + + yield client, [org, root, account, org_unit, policy] diff --git a/test/plugins/v0_1_0/aws/organizations/scp/__init__.py b/test/plugins/v0_1_0/aws/organizations/scp/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/plugins/v0_1_0/aws/organizations/scp/test_import_resources.py b/test/plugins/v0_1_0/aws/organizations/scp/test_import_resources.py new file mode 100644 index 000000000..76469e6cd --- /dev/null +++ b/test/plugins/v0_1_0/aws/organizations/scp/test_import_resources.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from iambic.plugins.v0_1_0.aws.handlers import import_organization_resources + + +class TestImportOrganizationResource: + @pytest.mark.asyncio + async def test_import_organization_resources( + self, + mock_execution_message, + mock_aws_config, + mock_fs, + mock_organizations_client, + ): + _ = mock_organizations_client + config = mock_aws_config() + _, templates_base_dir = mock_fs + + tasks = await import_organization_resources( + mock_execution_message(), config, templates_base_dir, [], remote_worker=None + ) + + tasks = await asyncio.gather(*tasks) + + assert tasks diff --git a/test/plugins/v0_1_0/aws/organizations/scp/test_template_generation.py b/test/plugins/v0_1_0/aws/organizations/scp/test_template_generation.py new file mode 100644 index 000000000..b8973b893 --- /dev/null +++ b/test/plugins/v0_1_0/aws/organizations/scp/test_template_generation.py @@ -0,0 +1,322 @@ +from __future__ import annotations + +import json +import os +from test.plugins.v0_1_0.aws.iam.policy.test_utils import ( # noqa: F401 # intentional for mocks + EXAMPLE_POLICY_DOCUMENT, +) +from test.plugins.v0_1_0.aws.organizations.scp.test_utils import ( + EXAMPLE_POLICY_DESCRIPTION, +) + +import pytest + +import iambic.plugins.v0_1_0.aws.organizations.scp.template_generation as template_generation +from iambic.plugins.v0_1_0.aws.event_bridge.models import ( + SCPMessageDetails as SCPPolicyMessageDetails, +) +from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig +from iambic.plugins.v0_1_0.aws.models import AWSOrganization +from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + AWS_SCP_POLICY_TEMPLATE, + AwsScpPolicyTemplate, + PolicyDocumentItem, + ServiceControlPolicyCache, + ServiceControlPolicyItem, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.template_generation import ( + collect_aws_scp_policies, + generate_scp_resource_files, + get_response_dir, + get_template_dir, + get_template_file_path, + upsert_templated_scp_policies, +) + +TEST_TEMPLATE_DIR = "resources/aws/organizations/scp" +TEST_TEMPLATE_PATH = f"{TEST_TEMPLATE_DIR}/example_policy.yaml" + + +def test_get_response_dir(mock_fs, mock_execution_message, mock_aws_account): + _, templates_base_dir = mock_fs + dir = get_response_dir(mock_execution_message(), mock_aws_account) + assert ( + dir + == f"{templates_base_dir}/.iambic/fake_execution_id/123456789012/organizations/scp" + ) + + +@pytest.mark.asyncio +def test_get_template_file_path(mock_aws_account, mock_aws_config, mock_fs): + _ = mock_aws_config() + aws_account = mock_aws_account + account_id = aws_account.account_id + aws_account_map = {aws_account.account_id: aws_account} + + _, templates_base_dir = mock_fs + resource_dir = get_template_dir(templates_base_dir) + + file_path = get_template_file_path( + resource_dir, + "PolicyName", + [aws_account_map[account_id].account_name], + aws_account_map, + ) + + assert file_path == "/".join( + [ + templates_base_dir, + TEST_TEMPLATE_DIR, + aws_account.account_name, + "policyname".lower() + ".yaml", + ] + ) + + +class TestCollectPolicies: + @pytest.mark.asyncio + async def test_collect_aws_scp_policies( + self, + mock_aws_config, + mock_aws_account, + mock_execution_message, + mocker, + mock_organizations_client, + mock_fs, + ): + config = mock_aws_config() + exe_message = mock_execution_message() + exe_message.provider_id = mock_aws_account.account_id + _, templates_base_dir = mock_fs + _, data = mock_organizations_client + policy = data[-1] + + spy_generate_scp_resource_files = mocker.spy( + template_generation, "generate_scp_resource_files" + ) + + await collect_aws_scp_policies( + exe_message=exe_message, + config=config, + scp_template_map={}, + detect_messages=[], + ) + + spy_generate_scp_resource_files.assert_called_once() + + output_path = f"{templates_base_dir}/.iambic/fake_execution_id/{exe_message.provider_id}/organizations/scp/output-{exe_message.provider_id}.json" + with open(output_path, "r") as f: + output = json.load(f) + assert len(output[0].get("policies")) == 1 + assert output[0].get("policies")[0].get("policy_id") == policy.get("Id") + + @pytest.mark.asyncio + async def test_detect_messages_when_create_policy( + self, + mock_aws_config, + mock_aws_account, + mock_execution_message, + mocker, + mock_organizations_client, + mock_fs, + ): + config = mock_aws_config() + exe_message = mock_execution_message() + exe_message.provider_id = mock_aws_account.account_id + client, _ = mock_organizations_client + _, templates_base_dir = mock_fs + + spy_generate_scp_resource_files = mocker.spy( + template_generation, "generate_scp_resource_files" + ) + + new_policy = client.create_policy( + Content=EXAMPLE_POLICY_DOCUMENT, + Description=EXAMPLE_POLICY_DESCRIPTION, + Name="PolicyToRemove", + Type="SERVICE_CONTROL_POLICY", + )["Policy"]["PolicySummary"] + + await collect_aws_scp_policies( + exe_message=exe_message, + config=config, + scp_template_map={}, + detect_messages=[ + SCPPolicyMessageDetails( + account_id=exe_message.provider_id, + policy_id=new_policy["Id"], + delete=False, + event="CreatePolicy", + ) + ], + ) + + output_path = f"{templates_base_dir}/.iambic/fake_execution_id/{exe_message.provider_id}/organizations/scp/output-{exe_message.provider_id}.json" + + with open(output_path, "r") as f: + output = json.load(f) + assert len(output[0].get("policies")) == 1 + assert output[0].get("policies")[0].get("policy_id") == new_policy["Id"] + + spy_generate_scp_resource_files.assert_called_once() + + @pytest.mark.asyncio + async def test_detect_messages_when_deleted_policy( + self, + mock_aws_config, + mock_aws_account, + mock_execution_message, + mocker, + mock_organizations_client, + mock_fs, + ): + config = mock_aws_config() + exe_message = mock_execution_message() + exe_message.provider_id = mock_aws_account.account_id + _ = mock_organizations_client + _ = mock_fs + + scp_item = ServiceControlPolicyItem( + Id="p-id", + Arn="arn:aws:organizations:::policy/p-id", + Name="OldPolicy", + Description="this is a new policy", + Type="SERVICE_CONTROL_POLICY", + AwsManaged=False, + Targets=[], + PolicyDocument=PolicyDocumentItem.parse_obj( + json.loads(EXAMPLE_POLICY_DOCUMENT) + ), + Tags=[], + ) + + ( + template_params, + template_properties, + ) = AwsScpPolicyTemplate.factory_template_props( + "123456789123", + scp_item, + AWSConfig(), + AWSOrganization( + org_id="o-id", + org_account_id="123456789123", + hub_role_arn="", + ), + ) + + policy = AwsScpPolicyTemplate( + **template_params, + properties=template_properties, + file_path="/", + ) + policy = mocker.Mock() + policy.properties.policy_id = template_properties.get("policy_id") + spy_resource_file_upsert = mocker.spy( + template_generation, "resource_file_upsert" + ) + await collect_aws_scp_policies( + exe_message=exe_message, + config=config, + scp_template_map={ + AWS_SCP_POLICY_TEMPLATE: {policy.properties.policy_id: policy} + }, + detect_messages=[ + SCPPolicyMessageDetails( + account_id=exe_message.provider_id, + policy_id=template_properties.get("policy_id"), + delete=True, + event="CreatePolicy", + ) + ], + ) + + policy.delete.asset_called_once() + spy_resource_file_upsert.assert_not_called() + + +@pytest.mark.asyncio +async def test_generate_scp_resource_files( + mock_aws_config, + mock_aws_account, + mock_execution_message, + mock_organizations_client, + mock_fs, +): + _ = mock_aws_config() + exe_message = mock_execution_message() + exe_message.provider_id = mock_aws_account.account_id + _, data = mock_organizations_client + _ = mock_fs + aws_account = mock_aws_account + + resource_files = await generate_scp_resource_files(exe_message, aws_account) + + assert len(resource_files.get("policies")) == 1 + with open(resource_files.get("policies")[0].get("file_path"), "r") as f: + output = json.load(f) + assert output.get("Id") == data[-1].get("Id") + + +@pytest.mark.asyncio +async def test_upsert_templated_scp_policies( + mock_aws_config, + mock_aws_account, + mock_fs, + mock_aws_organization, + mock_organizations_client, + mock_execution_message, +): + config = mock_aws_config() + aws_account = mock_aws_account + account_id = aws_account.account_id + aws_account_map = {aws_account.account_id: aws_account} + existing_template_map = {} + exe_message = mock_execution_message() + exe_message.provider_id = mock_aws_account.account_id + _, templates_base_dir = mock_fs + resource_dir = get_template_dir(templates_base_dir) + organization = mock_aws_organization + client, data = mock_organizations_client + template_path = f"{templates_base_dir}/.iambic/fake_execution_id/{exe_message.provider_id}/organizations/scp" + template_file_path = f"{template_path}/output-{exe_message.provider_id}.json" + + policy = ServiceControlPolicyItem( + Id="p-id", + Arn="arn:aws:organizations:::policy/p-id", + Name="OldPolicy", + Description="this is a new policy", + Type="SERVICE_CONTROL_POLICY", + AwsManaged=False, + Targets=[], + PolicyDocument=PolicyDocumentItem.parse_obj( + json.loads(EXAMPLE_POLICY_DOCUMENT) + ), + Tags=[], + ) + + os.makedirs(template_path) + file = open(template_file_path, "w") + json.dump(policy.dict(), file) + file.close() + + policy_cache = ServiceControlPolicyCache( + file_path=template_file_path, + policy_id=data[-1].get("Id"), + arn="arn:aws:organizations:::policy/p-id", + account_id=account_id, + ) + + template: AwsScpPolicyTemplate = await upsert_templated_scp_policies( + aws_account_map, + account_id, + policy_cache, # type: ignore + resource_dir, + existing_template_map, + config, + organization, + ) + + assert template.properties.policy_name == policy.Name + assert template.identifier == policy.Name + assert template.properties.policy_id == policy.Id + print(template) diff --git a/test/plugins/v0_1_0/aws/organizations/scp/test_utils.py b/test/plugins/v0_1_0/aws/organizations/scp/test_utils.py new file mode 100644 index 000000000..a33a4ffab --- /dev/null +++ b/test/plugins/v0_1_0/aws/organizations/scp/test_utils.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +import json + +import boto3 +import pytest +from botocore.exceptions import ClientError +from moto import mock_organizations +from moto.organizations.models import FakePolicy + +import iambic.plugins.v0_1_0.aws.organizations.scp.utils as utils +from iambic.core.models import ProposedChangeType +from iambic.core.utils import normalize_dict_keys, snake_to_camelcap +from iambic.plugins.v0_1_0.aws.iambic_plugin import AWSConfig +from iambic.plugins.v0_1_0.aws.models import AWSAccount, AWSOrganization +from iambic.plugins.v0_1_0.aws.organizations.scp.models import ( + AwsScpPolicyTemplate, + PolicyDocumentItem, + ServiceControlPolicyItem, + ServiceControlPolicyTargetItem, + TagItem, +) +from iambic.plugins.v0_1_0.aws.organizations.scp.utils import ( + apply_update_policy, + apply_update_policy_tags, + apply_update_policy_targets, + create_policy, + delete_policy, + get_policy, + list_policies, + service_control_policy_is_enabled, +) + +EXAMPLE_POLICY_NAME = "ServiceControlPolicyExample" +EXAMPLE_POLICY_DESCRIPTION = "Example description" +EXAMPLE_POLICY_DOCUMENT = """ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": [ + "lex:*" + ], + "Resource": "*", + "Condition": { + "StringNotEquals": { + "aws:RequestedRegion": "us-west-1" + } + } + } + ] +}""" + +EXAMPLE_TAGS = [ + dict( + Key="test_key", + Value="test_value", + ) +] + +EXAMPLE_ACCOUNT_NAME = "Account Name" +EXAMPLE_ACCOUNT_EMAIL = "account@noq.dev" + +EXAMPLE_ORGANIZATIONAL_UNIT_NAME = "Organizational Unit Name" + + +@pytest.fixture +def mock_organizations_client(monkeypatch): + with mock_organizations(): + client = boto3.client("organizations") + organization = client.create_organization(FeatureSet="ALL")["Organization"] + account = client.create_account( + AccountName=EXAMPLE_ACCOUNT_NAME, + Email=EXAMPLE_ACCOUNT_EMAIL, + )["CreateAccountStatus"] + + root = client.list_roots()["Roots"][0] + + org_unit = client.create_organizational_unit( + ParentId=root["Id"], Name=EXAMPLE_ORGANIZATIONAL_UNIT_NAME + )["OrganizationalUnit"] + + account_nested = client.create_account( + AccountName=EXAMPLE_ACCOUNT_NAME + "_nested", + Email=EXAMPLE_ACCOUNT_EMAIL + "_nested", + )["CreateAccountStatus"] + + client.move_account( + AccountId=account_nested.get("AccountId"), + SourceParentId=root.get("Id"), + DestinationParentId=org_unit.get("Id"), + ) + + _current_init = FakePolicy.__init__ + + def new_init(self, organization, **kwargs): + # CHECK pull request https://github.com/getmoto/moto/pull/6338 + self.tags = kwargs.get("Tags", {}) + _current_init(self, organization, **kwargs) + + FakePolicy.__init__ = new_init + + policy = client.create_policy( + Content=EXAMPLE_POLICY_DOCUMENT, + Description=EXAMPLE_POLICY_DESCRIPTION, + Name=EXAMPLE_POLICY_NAME, + Type="SERVICE_CONTROL_POLICY", + )["Policy"]["PolicySummary"] + + client.attach_policy(PolicyId=policy["Id"], TargetId=account["AccountId"]) + + client.tag_resource(ResourceId=policy["Id"], Tags=EXAMPLE_TAGS) + + yield client, [root, organization, account, org_unit, account_nested, policy] + + +@pytest.mark.asyncio +async def test_list_policies(mock_organizations_client, mocker): + client, data = mock_organizations_client + policy = data[-1] + + spy_list_targets_for_policy = mocker.spy(utils, "list_targets_for_policy") + spy_get_policy_statements = mocker.spy(utils, "get_policy_statements") + spy_list_tags_by_policy = mocker.spy(utils, "list_tags_by_policy") + spy_describe_policy = mocker.spy(utils, "describe_policy") + + resp = await list_policies(client) + + assert len(resp) == 1 + assert isinstance(resp[0], ServiceControlPolicyItem) + assert resp[0].Id == policy["Id"] + assert len(resp[0].Targets) == 1 + + spy_list_targets_for_policy.assert_called_once() + spy_get_policy_statements.assert_called_once() + spy_list_tags_by_policy.assert_called_once() + spy_describe_policy.assert_called_once() + + +@pytest.mark.asyncio +async def test_get_policy(mock_organizations_client): + client, data = mock_organizations_client + policy = data[-1] + + resp = await get_policy(client, policy["Id"]) + + assert isinstance(resp, ServiceControlPolicyItem) + assert resp.Id == policy["Id"] + assert len(resp.Targets) == 1 + + +@pytest.mark.asyncio +async def test_delete_policy(mock_organizations_client, mocker): + client, data = mock_organizations_client + root = data[0] + + policy = client.create_policy( + Content=EXAMPLE_POLICY_DOCUMENT, + Description=EXAMPLE_POLICY_DESCRIPTION, + Name="PolicyToDelete", + Type="SERVICE_CONTROL_POLICY", + )["Policy"]["PolicySummary"] + + org_unit = client.create_organizational_unit( + ParentId=root["Id"], Name=EXAMPLE_ORGANIZATIONAL_UNIT_NAME + )["OrganizationalUnit"] + + client.attach_policy( + PolicyId=policy["Id"], + TargetId=org_unit["Id"], + ) + + spy_detach_policy = mocker.spy(utils, "detach_policy") + spy_list_targets_for_policy = mocker.spy(utils, "list_targets_for_policy") + + await delete_policy(client, policy["Id"]) + + with pytest.raises(ClientError): + client.describe_policy(PolicyId=policy["Id"]) + + spy_detach_policy.assert_called_once() + spy_list_targets_for_policy.assert_called_once() + + +@pytest.mark.asyncio +async def test_create_policy(mock_organizations_client): + client, data = mock_organizations_client + + policy = await create_policy( + client, + dict( + PolicyDocument=EXAMPLE_POLICY_DOCUMENT, + Description="DESCRIPTION", + PolicyName="create-policy", + ), + ) + + assert client.describe_policy(PolicyId=policy["Id"]) + + +@pytest.mark.asyncio +async def test_service_control_policy_is_enabled(mock_organizations_client, mocker): + client, data = mock_organizations_client + spy_describe_organization = mocker.spy(utils, "describe_organization") + + assert await service_control_policy_is_enabled(client) + spy_describe_organization.assert_called_once() + + +@pytest.mark.asyncio +async def test_apply_update_policy(mock_organizations_client): + client, data = mock_organizations_client + scp_item = ServiceControlPolicyItem( + Id="p-id", + Arn="arn:aws:organizations:::policy/p-id", + Name="OldPolicy", + Description="this is a new policy", + Type="SERVICE_CONTROL_POLICY", + AwsManaged=False, + Targets=[], + PolicyDocument=PolicyDocumentItem.parse_obj( + json.loads(EXAMPLE_POLICY_DOCUMENT) + ), + Tags=[], + ) + + template_params, template_properties = AwsScpPolicyTemplate.factory_template_props( + "123456789123", + scp_item, + AWSConfig(), + AWSOrganization( + org_id="o-id", + org_account_id="123456789123", + hub_role_arn="", + ), + ) + policy = AwsScpPolicyTemplate( + **template_params, + properties=template_properties, + file_path="/", + ) + scp_item.Name = "NewPolicy" + changes = await apply_update_policy( + client, + normalize_dict_keys(policy.properties.dict(), snake_to_camelcap), + scp_item.dict(), + dict(resource_type="policy"), + ) + + assert len(changes) == 1 + assert changes[0].change_type == ProposedChangeType.UPDATE + + +@pytest.mark.asyncio +async def test_apply_update_policy_targets(mock_organizations_client): + client, data = mock_organizations_client + scp_item = ServiceControlPolicyItem( + Id="p-id", + Arn="arn:aws:organizations:::policy/p-id", + Name="OldPolicy", + Description="this is a new policy", + Type="SERVICE_CONTROL_POLICY", + AwsManaged=False, + Targets=[ + ServiceControlPolicyTargetItem( + TargetId="ou-123456789", + Arn="Asf", + Name="Asf", + ) + ], + PolicyDocument=PolicyDocumentItem.parse_obj( + json.loads(EXAMPLE_POLICY_DOCUMENT) + ), + Tags=[], + ) + + template_params, template_properties = AwsScpPolicyTemplate.factory_template_props( + "123456789123", + scp_item, + AWSConfig(), + AWSOrganization( + org_id="o-id", + org_account_id="123456789123", + hub_role_arn="", + ), + ) + + policy = AwsScpPolicyTemplate( + **template_params, + properties=template_properties, + file_path="/", + ) + + scp_item.Targets[0].TargetId = "123456789123" + + changes = await apply_update_policy_targets( + client, + normalize_dict_keys(policy.properties.dict(), snake_to_camelcap), + scp_item.dict(), + dict(resource_type="policy"), + AWSAccount(account_name="test"), + ) + + assert len(changes) == 2 + assert changes[0].change_type == ProposedChangeType.DETACH + assert changes[1].change_type == ProposedChangeType.ATTACH + + +@pytest.mark.asyncio +async def test_apply_update_policy_tags(mock_organizations_client): + client, data = mock_organizations_client + scp_item = ServiceControlPolicyItem( + Id="p-id", + Arn="arn:aws:organizations:::policy/p-id", + Name="OldPolicy", + Description="this is a new policy", + Type="SERVICE_CONTROL_POLICY", + AwsManaged=False, + Targets=[], + PolicyDocument=PolicyDocumentItem.parse_obj( + json.loads(EXAMPLE_POLICY_DOCUMENT) + ), + Tags=[TagItem(Key="key", Value="value")], + ) + + template_params, template_properties = AwsScpPolicyTemplate.factory_template_props( + "123456789123", + scp_item, + AWSConfig(), + AWSOrganization( + org_id="o-id", + org_account_id="123456789123", + hub_role_arn="", + ), + ) + + policy = AwsScpPolicyTemplate( + **template_params, + properties=template_properties, + file_path="/", + ) + + scp_item.Tags[0].Key = "newKey" + + changes = await apply_update_policy_tags( + client, + normalize_dict_keys(policy.properties.dict(), snake_to_camelcap), + scp_item.dict(), + dict(resource_type="policy"), + AWSAccount(account_name="test"), + ) + + assert len(changes) == 2 + assert changes[0].change_type == ProposedChangeType.DETACH + assert changes[1].change_type == ProposedChangeType.ATTACH diff --git a/test/plugins/v0_1_0/aws/test_event_bridge.py b/test/plugins/v0_1_0/aws/test_event_bridge.py new file mode 100644 index 000000000..5983a24e0 --- /dev/null +++ b/test/plugins/v0_1_0/aws/test_event_bridge.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import pytest + +from iambic.plugins.v0_1_0.aws.event_bridge.models import SCPMessageDetails + + +class TestSCPMessageDetails: + @pytest.mark.parametrize( + "resource, org, value", + [ + ("TagResource", "organizations.amazonaws.com", True), + ("UntagResource", "organizations.amazonaws.com", True), + ("TagResource", "otherservice.amazonaws.com", False), + ("UntagResource", "otherservice.amazonaws.com", False), + ("OtherOperation", "organizations.amazonaws.com", False), + ], + ) + def test_tag_event(self, resource, org, value): + assert SCPMessageDetails.tag_event(resource, org) == value + + @pytest.mark.parametrize( + "request_params, response_elements, value", + [ + ({"policyId": "foo"}, None, "foo"), + (None, {"policy": {"policySummary": {"id": "bar"}}}, "bar"), + (None, None, None), + ], + ) + def test_get_policy_id(self, request_params, response_elements, value): + assert ( + SCPMessageDetails.get_policy_id(request_params, response_elements) == value + ) From 5432eaf4344c41636a6f9caa1ca1fe7f8d6f53c0 Mon Sep 17 00:00:00 2001 From: Curtis Castrapel Date: Fri, 26 May 2023 13:19:28 -0700 Subject: [PATCH 2/2] Clean up log messages --- .../plugins/v0_1_0/aws/organizations/scp/utils.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py b/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py index f2ed8bce6..6d1c1ad02 100644 --- a/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py +++ b/iambic/plugins/v0_1_0/aws/organizations/scp/utils.py @@ -168,13 +168,12 @@ async def get_policy(client, policyId: str) -> ServiceControlPolicyItem: wait=wait_exponential(multiplier=1, min=4, max=15), ) async def detach_policy(client, policyId, targetId): - log.info(f"Detaching policy {policyId} from {targetId}") await boto_crud_call( client.detach_policy, PolicyId=policyId, TargetId=targetId, ) - log.info(f"Detached policy {policyId} from {targetId}") + log.debug(f"Detached policy {policyId} from {targetId}") @retry( @@ -202,11 +201,9 @@ async def delete_policy(client, policyId: str, *args, **kwargs): await asyncio.gather(*targets_tasks) - log.info(f"Deleting policy {policyId}") - await boto_crud_call(client.delete_policy, PolicyId=policyId) - log.info(f"Deleted policy {policyId}") + log.debug(f"Deleted policy {policyId}") @retry( @@ -492,7 +489,7 @@ def __apply_tags( Tags=tags_to_apply, ) tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) - log.info(log_str, tags=tags_to_apply, **log_params) + log.debug(log_str, tags=tags_to_apply, **log_params) return tasks, response @@ -535,7 +532,7 @@ def __remove_tags(client, policy, current_policy, log_params): TagKeys=tags_to_remove, ) tasks.append(plugin_apply_wrapper(apply_awaitable, proposed_changes)) - log.info(log_str, tags=tags_to_remove, **log_params) + log.debug(log_str, tags=tags_to_remove, **log_params) return tasks, response @@ -600,7 +597,7 @@ async def attach_policies(): tasks.append(plugin_apply_wrapper(attach_policies(), proposed_changes)) - log.info(log_str, tags=targets_to_apply, **log_params) + log.debug(log_str, tags=targets_to_apply, **log_params) return tasks, response @@ -668,6 +665,6 @@ async def detach_policies(): ) tasks.append(plugin_apply_wrapper(detach_policies(), proposed_changes)) - log.info(log_str, tags=targets_to_remove, **log_params) + log.debug(log_str, tags=targets_to_remove, **log_params) return tasks, response