Feat/nested ou targets (#538)

* Adding in additional tests for account processing

* Adding in unit tests for organisations abstraction

* Adding in unit tests for organisations abstraction

* Initial commit - Now optionally resolves children OUs

* Linting

* starting exclusion work

* Adding in unit tests for schema validation

* Adding in an example config to user-guide.md

* Codereview comments

* Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py

Co-authored-by: Simon Kok

* Apply suggestions from code review

* Remove redundant pylint disable line

---------

Co-authored-by: Stewart Wallace
Co-authored-by: Simon Kok

docs/user-guide.md

@@ -271,10 +271,16 @@ targets:
       # stages, each stage containing up to X accounts
       size: 30
     exclude:
-      # (Optional) List of accounts to exclude from this target.
-      # Currently only supports account Ids
+      # (Optional) List of accounts to exclude from this path.
       - 9999999999
     properties: ...
+  - path: /my_ou/production/**/*  # This would target all accounts in the OU path
+    regions: [eu-central-1, us-west-1]
+    name: production_step
+    exclude: # (Optional) List of OU Paths and Account Ids
+      - /my-ou/production/alpha # excludes any accounts and OUs in this path.
+      - 11111111111 # Excludes this account, regardless of OU path.
+    properties: ...
 ```
 
 CodePipeline has a limit of 50 actions per stage.

requirements-dev.txt

@@ -3,5 +3,6 @@ isort==5.12.0
 mock==5.1.0
 pylint==2.17.4
 pytest~=7.4.0
+pytest-cov==3.0.0
 tox==3.28.0
 yamllint==1.32.0

src/lambda_codebase/account_processing/configure_account_tags.py

@@ -7,37 +7,44 @@
 Will not delete tags that aren't
 in the config file.
 """
-
-from organizations import Organizations
-
 import boto3
 from aws_xray_sdk.core import patch_all
 from logger import configure_logger
 
 patch_all()
 LOGGER = configure_logger(__name__)
+ORG_CLIENT = boto3.client("organizations")
 
 
-def create_account_tags(account_id, tags, org_session: Organizations):
+def create_account_tags(account_id, tags, client):
     LOGGER.info(
         "Ensuring Account: %s has tags: %s",
         account_id,
         tags,
     )
-    org_session.create_account_tags(account_id, tags)
+    formatted_tags = [
+        {"Key": str(key), "Value": str(value)}
+        for tag in tags
+        for key, value in tag.items()
+    ]
+    LOGGER.debug(
+        "Ensuring Account: %s has tags (formatted): %s",
+        account_id,
+        formatted_tags,
+    )
+    client.tag_resource(ResourceId=account_id, Tags=formatted_tags)
 
 
 def lambda_handler(event, _):
     if event.get("tags"):
-        organizations = Organizations(boto3)
         create_account_tags(
             event.get("account_id"),
             event.get("tags"),
-            organizations,
+            ORG_CLIENT,
         )
     else:
         LOGGER.info(
             "Account: %s does not need tags configured",
-            event.get('account_full_name'),
+            event.get("account_full_name"),
         )
     return event

src/lambda_codebase/account_processing/get_account_regions.py

@@ -17,31 +17,33 @@
 AWS_PARTITION = os.getenv("AWS_PARTITION")
 
 
+def get_default_regions_for_account(ec2_client):
+    filtered_regions = ec2_client.describe_regions(
+        AllRegions=False,
+        Filters=[
+            {
+                "Name": "opt-in-status",
+                "Values": [
+                    "opt-in-not-required",
+                    "opted-in",
+                ],
+            },
+        ],
+    )["Regions"]
+    default_regions = [region["RegionName"] for region in filtered_regions]
+    return default_regions
+
+
 def lambda_handler(event, _):
-    LOGGER.info("Fetching Default regions %s", event.get('account_full_name'))
+    LOGGER.info("Fetching Default regions %s", event.get("account_full_name"))
     sts = STS()
     account_id = event.get("account_id")
     role = sts.assume_cross_account_role(
         f"arn:{AWS_PARTITION}:iam::{account_id}:role/{ADF_ROLE_NAME}",
         "adf_account_get_regions",
     )
+    default_regions = get_default_regions_for_account(role.client("ec2"))
 
-    ec2_client = role.client("ec2")
-    default_regions = [
-        region["RegionName"]
-        for region in ec2_client.describe_regions(
-            AllRegions=False,
-            Filters=[
-                {
-                    "Name": "opt-in-status",
-                    "Values": [
-                        "opt-in-not-required",
-                        "opted-in"
-                    ],
-                }
-            ],
-        )["Regions"]
-    ]
     LOGGER.debug("Default regions for %s: %s", account_id, default_regions)
     return {
         **event,

src/lambda_codebase/account_processing/pytest.ini

@@ -1,2 +1,7 @@
 [pytest]
 testpaths = tests
+addopts = --cov=./src/lambda_codebase/account_processing/  --cov-fail-under=50 --cov-report term
+
+[coverage:run]
+omit = tests/
+

src/lambda_codebase/account_processing/tests/test_account_tags.py

@@ -0,0 +1,62 @@
+"""
+Tests the account tag configuration lambda
+"""
+
+import unittest
+import boto3
+from botocore.stub import Stubber
+from aws_xray_sdk import global_sdk_config
+from ..configure_account_tags import (
+    create_account_tags,
+)
+
+global_sdk_config.set_sdk_enabled(False)
+
+# pylint: disable=W0106
+class SuccessTestCase(unittest.TestCase):
+    def test_account_tag_creation(self):
+        test_event = {"account_id": "123456789012", "tags": [{"CreatedBy": "ADF"}]}
+        ou_client = boto3.client("organizations")
+        stubber = Stubber(ou_client)
+        stubber.add_response(
+            "tag_resource",
+            {},
+            {
+                "Tags": [{"Key": "CreatedBy", "Value": "ADF"}],
+                "ResourceId": "123456789012",
+            },
+        ),
+        stubber.activate()
+        create_account_tags(
+            test_event.get("account_id"), test_event.get("tags"), ou_client
+        )
+        stubber.assert_no_pending_responses()
+
+    def test_account_tag_creation_multiple_tags(self):
+        test_event = {
+            "account_id": "123456789012",
+            "tags": [
+                {
+                    "CreatedBy": "ADF",
+                    "TagName": "TagValue",
+                }
+            ],
+        }
+        ou_client = boto3.client("organizations")
+        stubber = Stubber(ou_client)
+        stubber.add_response(
+            "tag_resource",
+            {},
+            {
+                "Tags": [
+                    {"Key": "CreatedBy", "Value": "ADF"},
+                    {"Key": "TagName", "Value": "TagValue"},
+                ],
+                "ResourceId": "123456789012",
+            },
+        ),
+        stubber.activate()
+        create_account_tags(
+            test_event.get("account_id"), test_event.get("tags"), ou_client
+        )
+        stubber.assert_no_pending_responses()

src/lambda_codebase/account_processing/tests/test_get_default_regions.py

@@ -0,0 +1,62 @@
+"""
+Tests the account tag configuration lambda
+"""
+
+import unittest
+import boto3
+from botocore.stub import Stubber
+from aws_xray_sdk import global_sdk_config
+from ..get_account_regions import (
+    get_default_regions_for_account,
+)
+
+global_sdk_config.set_sdk_enabled(False)
+
+# pylint: disable=W0106
+class SuccessTestCase(unittest.TestCase):
+    @staticmethod
+    def test_get_default_regions_for_account():
+        ec2_client = boto3.client("ec2")
+        stubber = Stubber(ec2_client)
+        stubber.add_response(
+            "describe_regions",
+            {
+                "Regions": [
+                    {
+                        "Endpoint": "blah",
+                        "RegionName": "us-east-1",
+                        "OptInStatus": "opt-in-not-required",
+                    },
+                    {
+                        "Endpoint": "blah",
+                        "RegionName": "us-east-2",
+                        "OptInStatus": "opt-in-not-required",
+                    },
+                    {
+                        "Endpoint": "blah",
+                        "RegionName": "us-east-3",
+                        "OptInStatus": "opted-in",
+                    },
+                    {
+                        "Endpoint": "blah",
+                        "RegionName": "us-east-4",
+                        "OptInStatus": "opted-in",
+                    },
+                ]
+            },
+            {
+                "AllRegions": False,
+                "Filters": [
+                    {
+                        "Values": [
+                            "opt-in-not-required",
+                            "opted-in",
+                        ],
+                        "Name": "opt-in-status",
+                    },
+                ],
+            },
+        ),
+        stubber.activate()
+        regions = get_default_regions_for_account(ec2_client)
+        assert regions == ["us-east-1", "us-east-2", "us-east-3", "us-east-4"]