feat: adding support for non cloudtrail modes of c7n

pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "aws-service-catalog-puppet"
-version = "0.231.1"
+version = "0.232.0"
 description = "Making it easier to deploy ServiceCatalog products"
 classifiers = ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Natural Language :: English"]
 homepage = "https://service-catalog-tools-workshop.com/"

servicecatalog_puppet/commands/manifest.py

@@ -122,6 +122,7 @@ def expand(f, puppet_account_id, regions, single_account, subset=None):
     new_manifest = manifest_utils.rewrite_stacks(new_manifest, puppet_account_id)
     new_manifest = manifest_utils.rewrite_scps(new_manifest, puppet_account_id)
     new_manifest = manifest_utils.parse_conditions(new_manifest)
+    new_manifest = manifest_utils.rewrite_c7n_cloudtrails(new_manifest)
 
     if subset and subset.get("section"):
         click.echo(f"Filtering for subset: {subset}")

servicecatalog_puppet/commands/task_reference_helpers/generators/c7n_aws_cloudtrails.py → servicecatalog_puppet/commands/task_reference_helpers/generators/c7n_aws_lambdas.py

@@ -47,6 +47,7 @@ def prepare_account_as_hub(
     custodian_region,
     custodian_role_name,
     custodian_role_path,
+    schedule_expression,
     custodian_c7n_version,
     organization,
 ):
@@ -63,6 +64,7 @@ def prepare_account_as_hub(
         organization=organization,
         role_name=custodian_role_name,
         role_path=custodian_role_path,
+        schedule_expression=schedule_expression,
         dependencies_by_reference=[],
         execution=task_to_add.get("execution"),
         **manifest_related_args(task_to_add),
@@ -114,7 +116,7 @@ def forward_events_region_task(
     )
 
 
-def deploy_policies(
+def handle_deploy_policies(
     task_to_add,
     all_tasks,
     account_id,
@@ -151,14 +153,15 @@ def deploy_policies(
     deploy_policies_task["deployments"][region].append(account_id)
 
 
-def handle_c7n_aws_cloudtrails_for_custodian(
+def handle_c7n_aws_lambdas_for_custodian(
     task_to_add,
     all_tasks,
     region,
     custodian_account_id,
     custodian_region,
     custodian_role_name,
     custodian_role_path,
+    schedule_expression,
     custodian_role_managed_policy_arns,
     custodian_c7n_version,
     organization,
@@ -180,6 +183,7 @@ def handle_c7n_aws_cloudtrails_for_custodian(
         custodian_region,
         custodian_role_name,
         custodian_role_path,
+        schedule_expression,
         custodian_c7n_version,
         organization,
     )
@@ -200,7 +204,7 @@ def handle_c7n_aws_cloudtrails_for_custodian(
     )
 
 
-def handle_c7n_aws_cloudtrails_for_spoke(
+def handle_c7n_aws_lambdas_for_spoke(
     task_to_add,
     all_tasks,
     account_id,
@@ -246,7 +250,7 @@ def get_custodian_region(manifest, account_id):
             return a["default_region"]
 
 
-def handle_c7n_aws_cloudtrails(
+def handle_c7n_aws_lambdas(
     all_tasks,
     all_tasks_task_reference,
     item_name,
@@ -261,24 +265,26 @@ def handle_c7n_aws_cloudtrails(
     custodian_account_id = task_to_add.get("custodian")
     custodian_role_name = task_to_add.get("role_name")
     custodian_role_path = task_to_add.get("role_path")
+    schedule_expression = task_to_add.get("schedule_expression", "")
     custodian_role_managed_policy_arns = task_to_add.get("role_managed_policy_arns")
     custodian_c7n_version = task_to_add.get("c7n_version")
     organization = get_organization_for_account(manifest, account_id)
     custodian_region = get_custodian_region(manifest, custodian_account_id)
 
-    handle_c7n_aws_cloudtrails_for_custodian(
+    handle_c7n_aws_lambdas_for_custodian(
         task_to_add,
         all_tasks,
         region,
         custodian_account_id,
         custodian_region,
         custodian_role_name,
         custodian_role_path,
+        schedule_expression,
         custodian_role_managed_policy_arns,
         custodian_c7n_version,
         organization,
     )
-    handle_c7n_aws_cloudtrails_for_spoke(
+    handle_c7n_aws_lambdas_for_spoke(
         task_to_add,
         all_tasks,
         account_id,
@@ -290,7 +296,7 @@ def handle_c7n_aws_cloudtrails(
         custodian_role_managed_policy_arns,
     )
 
-    deploy_policies(
+    handle_deploy_policies(
         task_to_add,
         all_tasks,
         account_id,

servicecatalog_puppet/commands/task_reference_helpers/generators/generator.py

@@ -2,7 +2,7 @@
 #  SPDX-License-Identifier: Apache-2.0
 from servicecatalog_puppet import constants
 from servicecatalog_puppet.commands.task_reference_helpers.generators import (
-    c7n_aws_cloudtrails,
+    c7n_aws_lambdas,
     imported_portfolios,
     launches,
     organizational_units,
@@ -113,8 +113,8 @@ def generate(
             task_to_add,
         )
 
-    if section_name == constants.C7N_AWS_CLOUDTRAILS:
-        c7n_aws_cloudtrails.handle_c7n_aws_cloudtrails(
+    if section_name == constants.C7N_AWS_LAMBDAS:
+        c7n_aws_lambdas.handle_c7n_aws_lambdas(
             all_tasks,
             all_tasks_task_reference,
             item_name,

servicecatalog_puppet/constants.py

@@ -47,6 +47,8 @@
 TAG_POLICY = "tag-policy"
 C7N_AWS_CLOUDTRAILS = "c7n-aws-cloudtrails"
 C7N_AWS_CLOUDTRAIL = "c7n-aws-cloudtrail"
+C7N_AWS_LAMBDAS = "c7n-aws-lambdas"
+C7N_AWS_LAMBDA = "c7n-aws-lambda"
 
 ORGANIZATIONAL_UNIT = "organizational-unit"
 ORGANIZATIONAL_UNITS = "organizational-units"
@@ -152,7 +154,7 @@
     SERVICE_CONTROL_POLICIES,
     SIMULATE_POLICIES,
     TAG_POLICIES,
-    C7N_AWS_CLOUDTRAILS,
+    C7N_AWS_LAMBDAS,
 ]
 ALL_SECTION_NAMES_THAT_GENERATE_OUTPUTS = [
     LAUNCHES,
@@ -186,7 +188,7 @@
     (SIMULATE_POLICY, SIMULATE_POLICIES),
     (TAG_POLICY, TAG_POLICIES),
     (ORGANIZATIONAL_UNIT, ORGANIZATIONAL_UNITS),
-    (C7N_AWS_CLOUDTRAIL, C7N_AWS_CLOUDTRAILS),
+    (C7N_AWS_LAMBDA, C7N_AWS_LAMBDAS),
 ]
 
 SECTION_NAME_SINGULAR_AND_PLURAL_LIST_THAT_SUPPORTS_PARAMETERS = [
@@ -219,7 +221,7 @@
     WORKSPACE: WORKSPACES,
     SIMULATE_POLICY: SIMULATE_POLICIES,
     SERVICE_CONTROL_POLICY: SERVICE_CONTROL_POLICIES,
-    C7N_AWS_CLOUDTRAIL: C7N_AWS_CLOUDTRAILS,
+    C7N_AWS_LAMBDA: C7N_AWS_LAMBDAS,
 }
 
 CODEBUILD_DEFAULT_IMAGE = "aws/codebuild/standard:5.0"
@@ -276,7 +278,7 @@
     TAG_POLICIES,
     SIMULATE_POLICIES,
     ASSERTIONS,
-    C7N_AWS_CLOUDTRAILS,
+    C7N_AWS_LAMBDAS,
 ]
 
 SERVICE_CATALOG_PUPPET_MANIFEST_SSM_PREFIX = "/servicecatalog-puppet/manifest"
@@ -294,7 +296,7 @@
     SERVICE_CONTROL_POLICIES: "apply_to",
     SIMULATE_POLICIES: "simulate_for",
     TAG_POLICIES: "apply_to",
-    C7N_AWS_CLOUDTRAILS: "apply_to",
+    C7N_AWS_LAMBDAS: "apply_to",
 }
 
 CLOUDFORMATION_HAPPY_STATUS = [

servicecatalog_puppet/constants_unit_test.py

@@ -64,7 +64,7 @@ def test_constants_values():
         constants.SERVICE_CONTROL_POLICIES,
         constants.SIMULATE_POLICIES,
         constants.TAG_POLICIES,
-        constants.C7N_AWS_CLOUDTRAILS,
+        constants.C7N_AWS_LAMBDAS,
     ]
     assert constants.RESULTS_DIRECTORY == "results"
     assert constants.NO_CHANGE == "NO_CHANGE"
@@ -145,7 +145,7 @@ def test_constants_values():
         (constants.SIMULATE_POLICY, constants.SIMULATE_POLICIES),
         (constants.TAG_POLICY, constants.TAG_POLICIES),
         (constants.ORGANIZATIONAL_UNIT, constants.ORGANIZATIONAL_UNITS),
-        (constants.C7N_AWS_CLOUDTRAIL, constants.C7N_AWS_CLOUDTRAILS),
+        (constants.C7N_AWS_LAMBDA, constants.C7N_AWS_LAMBDAS),
     ]
     assert constants.SECTION_NAME_SINGULAR_AND_PLURAL_LIST_THAT_SUPPORTS_PARAMETERS == [
         (constants.LAUNCH, constants.LAUNCHES),
@@ -167,5 +167,5 @@ def test_constants_values():
         constants.WORKSPACE: constants.WORKSPACES,
         constants.SIMULATE_POLICY: constants.SIMULATE_POLICIES,
         constants.SERVICE_CONTROL_POLICY: constants.SERVICE_CONTROL_POLICIES,
-        constants.C7N_AWS_CLOUDTRAIL: constants.C7N_AWS_CLOUDTRAILS,
+        constants.C7N_AWS_LAMBDA: constants.C7N_AWS_LAMBDAS,
     }

servicecatalog_puppet/manifest_utils.py

@@ -79,7 +79,7 @@ def load(f, puppet_account_id):
         constants.SERVICE_CONTROL_POLICIES: {},
         constants.SIMULATE_POLICIES: {},
         constants.TAG_POLICIES: {},
-        constants.C7N_AWS_CLOUDTRAILS: {},
+        constants.C7N_AWS_LAMBDAS: {},
     }
     intrinsic_functions_map = get_intrinsic_functions_map(
         manifest_name, puppet_account_id
@@ -764,7 +764,7 @@ def get_tasks_for(
             "tag-policies": "apply_to",
             "simulate-policies": "simulate_for",
             constants.ORGANIZATIONAL_UNITS: "create_in",
-            constants.C7N_AWS_CLOUDTRAILS: "apply_to",
+            constants.C7N_AWS_LAMBDAS: "apply_to",
         }.get(section_name)
 
         if (
@@ -918,7 +918,7 @@ def get_tasks_for(
                 name=item.get("name"),
                 tags=item.get("tags"),
             ),
-            constants.C7N_AWS_CLOUDTRAILS: dict(
+            constants.C7N_AWS_LAMBDAS: dict(
                 execution=item.get("execution", constants.EXECUTION_MODE_DEFAULT),
                 policies=item.get("policies"),
                 custodian=item.get("custodian"),
@@ -928,6 +928,7 @@ def get_tasks_for(
                 role_path=item.get(
                     "role_path", constants.C7N_CUSTODIAN_ROLE_PATH_DEFAULT
                 ),
+                schedule_expression=item.get("schedule_expression", ""),
                 role_managed_policy_arns=item.get(
                     "role_managed_policy_arns",
                     constants.C7N_CUSTODIAN_MANAGED_POLICY_ARNS_DEFAULT,
@@ -979,7 +980,7 @@ def get_tasks_for(
                     ),
                     "tag-policies": dict(account_id=account_id, ou_name="",),
                     constants.SIMULATE_POLICIES: dict(account_id=account_id,),
-                    constants.C7N_AWS_CLOUDTRAILS: dict(account_id=account_id,),
+                    constants.C7N_AWS_LAMBDAS: dict(account_id=account_id,),
                 }.get(section_name)
                 if tag_name in account.get("tags"):
                     if isinstance(regions, str):
@@ -1072,7 +1073,7 @@ def get_tasks_for(
                 "tag-policies": dict(account_id=account_id, ou_name="",),
                 constants.SIMULATE_POLICIES: dict(account_id=account_id,),
                 constants.ORGANIZATIONAL_UNITS: dict(account_id=account_id,),
-                constants.C7N_AWS_CLOUDTRAILS: dict(account_id=account_id,),
+                constants.C7N_AWS_LAMBDAS: dict(account_id=account_id,),
             }.get(section_name)
 
             if isinstance(regions, str):
@@ -1578,6 +1579,16 @@ def parse_conditions(manifest):
     return manifest
 
 
+def rewrite_c7n_cloudtrails(manifest):
+    if manifest.get(constants.C7N_AWS_CLOUDTRAILS):
+        manifest[constants.C7N_AWS_LAMBDAS] = manifest[constants.C7N_AWS_CLOUDTRAILS]
+        del manifest[constants.C7N_AWS_CLOUDTRAILS]
+        for item_name, item in manifest[constants.C7N_AWS_LAMBDAS].items():
+            for policy in item["policies"]:
+                policy["mode"]["type"] = "cloudtrail"
+    return manifest
+
+
 def rewrite_organizational_units(manifest, conversions, client):
     for item_name, item in manifest.get(constants.ORGANIZATIONAL_UNITS, {}).items():
         path = item.get("path")

servicecatalog_puppet/workflow/c7n/deploy_c7n_policies.py

@@ -1,11 +1,9 @@
 #  Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #  SPDX-License-Identifier: Apache-2.0
 
-from datetime import datetime
-
 import luigi
 
-from servicecatalog_puppet import constants, serialisation_utils
+from servicecatalog_puppet import constants, serialisation_utils, config
 from servicecatalog_puppet.serialisation_utils import unwrap
 from servicecatalog_puppet.workflow.dependencies import tasks
 
@@ -24,43 +22,51 @@ def params_for_results_display(self):
 
     def run(self):
         policies = list()
-        member_role = "arn:aws:iam::{account_id}:role" + self.role_path + self.role_name
+        partition = config.get_partition()
+        member_role = (
+            "arn:"
+            + partition
+            + ":iam::{account_id}:role"
+            + self.role_path
+            + self.role_name
+        )
         for policy in unwrap(self.policies):
-            policy["mode"]["type"] = "cloudtrail"
-            policy["mode"]["member-role"] = member_role
+            if policy.get("mode", {}).get("type") == "cloudtrail":
+                policy["mode"]["member-role"] = member_role
             policies.append(policy)
 
         bucket = f"sc-puppet-c7n-artifacts-{self.account_id}-{self.region}"
-        key = str(datetime.now())
+        key = "latest"
 
         with self.spoke_regional_client("s3") as s3:
             s3.put_object(
                 Bucket=bucket,
                 Key=key,
                 Body=serialisation_utils.dump(unwrap(dict(policies=policies))),
             )
-            cached_output_signed_url = s3.generate_presigned_url(
-                "get_object",
-                Params={"Bucket": bucket, "Key": key},
-                ExpiresIn=60 * 60 * 24,
-            )
-        policies_file_url = cached_output_signed_url
         custodian_role_arn = (
-            f"arn:aws:iam::{self.account_id}:role" + self.role_path + self.role_name
+            f"arn:{partition}:iam::{self.account_id}:role"
+            + self.role_path
+            + self.role_name
         )
         regions_to_run_in = list(self.deployments.keys())
-        parameters_to_use = [
-            dict(name="POLICIES_FILE_URL", value=policies_file_url, type="PLAINTEXT",),
-            dict(name="REGIONS", value=" ".join(regions_to_run_in), type="PLAINTEXT",),
-            dict(
-                name="CUSTODIAN_ROLE_ARN", value=custodian_role_arn, type="PLAINTEXT",
-            ),
-        ]
 
+        with self.spoke_client("ssm") as ssm:
+            ssm.put_parameter(
+                Name="/servicecatalog-puppet/aws-c7n-lambdas/REGIONS",
+                Value=" ".join(regions_to_run_in),
+                Type="String",
+                Overwrite=True,
+            )
+            ssm.put_parameter(
+                Name="/servicecatalog-puppet/aws-c7n-lambdas/CUSTODIAN_ROLE_ARN",
+                Value=custodian_role_arn,
+                Type="String",
+                Overwrite=True,
+            )
         with self.spoke_client("codebuild") as codebuild:
             result = codebuild.start_build_and_wait_for_completion(
                 projectName="servicecatalog-puppet-deploy-c7n",
-                environmentVariablesOverride=parameters_to_use,
             )
             if result.get("buildStatus") != "SUCCEEDED":
                 raise Exception(f"Deploying policy failed: {result.get('buildStatus')}")