Updating codebuild image to v5 and fixed issue in task_topological_ge…

…nerations_without_scheduler where it would exit early (#642)

pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "aws-service-catalog-puppet"
-version = "0.222.0"
+version = "0.223.0"
 description = "Making it easier to deploy ServiceCatalog products"
 classifiers = ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Natural Language :: English"]
 homepage = "https://service-catalog-tools-workshop.com/"

servicecatalog_puppet/constants.py

@@ -212,7 +212,7 @@
     SERVICE_CONTROL_POLICY: SERVICE_CONTROL_POLICIES,
 }
 
-CODEBUILD_DEFAULT_IMAGE = "aws/codebuild/standard:4.0"
+CODEBUILD_DEFAULT_IMAGE = "aws/codebuild/standard:5.0"
 
 DEFAULT_TERRAFORM_VERSION_PARAMETER_NAME = (
     "/servicecatalog-puppet/terraform/default-version"

servicecatalog_puppet/servicecatalog-puppet-initialiser.template.yaml

@@ -16,6 +16,7 @@ Metadata:
           - ShouldCollectCloudformationEvents
           - ShouldForwardEventsToEventbridge
           - ShouldForwardFailuresToOpscenter
+          - CodeBuildEnvironmentImage
       - Label:
           default: "Options"
         Parameters:
@@ -85,6 +86,9 @@ Metadata:
 
 
 Parameters:
+  CodeBuildEnvironmentImage:
+    Type: String
+    Default: "aws/codebuild/standard:5.0"
   EnabledRegions:
     Type: String
     Description: |
@@ -300,7 +304,7 @@ Resources:
       Environment:
         Type: linuxContainer
         ComputeType: BUILD_GENERAL1_SMALL
-        Image: aws/codebuild/standard:4.0
+        Image: !Ref CodeBuildEnvironmentImage
         EnvironmentVariables:
           - Name: VERSION
             Type: PLAINTEXT

servicecatalog_puppet/servicecatalog-puppet-spoke.template.yaml

@@ -137,7 +137,7 @@ Resources:
       Environment:
         Type: linuxContainer
         ComputeType: !Ref DeployEnvironmentComputeType
-        Image: aws/codebuild/standard:4.0
+        Image: aws/codebuild/standard:5.0
         EnvironmentVariables:
           - Name: VERSION
             Type: PLAINTEXT

servicecatalog_puppet/template_builder/hub/bootstrap.py

@@ -760,7 +760,7 @@ def get_template(
                 TimeoutInMinutes=60,
                 Environment=codebuild.Environment(
                     ComputeType="BUILD_GENERAL1_SMALL",
-                    Image="aws/codebuild/standard:4.0",
+                    Image=constants.CODEBUILD_DEFAULT_IMAGE,
                     Type="LINUX_CONTAINER",
                     EnvironmentVariables=[
                         {
@@ -1005,7 +1005,7 @@ def get_template(
         TimeoutInMinutes=480,
         Environment=codebuild.Environment(
             ComputeType=t.Ref(deploy_environment_compute_type_parameter),
-            Image="aws/codebuild/standard:4.0",
+            Image=constants.CODEBUILD_DEFAULT_IMAGE,
             Type="LINUX_CONTAINER",
             EnvironmentVariables=[
                 {
@@ -1055,7 +1055,7 @@ def get_template(
             TimeoutInMinutes=60,
             Environment=codebuild.Environment(
                 ComputeType="BUILD_GENERAL1_SMALL",
-                Image="aws/codebuild/standard:4.0",
+                Image=constants.CODEBUILD_DEFAULT_IMAGE,
                 Type="LINUX_CONTAINER",
                 EnvironmentVariables=[
                     {"Type": "PLAINTEXT", "Name": "OU_OR_PATH", "Value": "CHANGE_ME"},
@@ -1086,7 +1086,7 @@ def get_template(
             TimeoutInMinutes=60,
             Environment=codebuild.Environment(
                 ComputeType="BUILD_GENERAL1_SMALL",
-                Image="aws/codebuild/standard:4.0",
+                Image=constants.CODEBUILD_DEFAULT_IMAGE,
                 Type="LINUX_CONTAINER",
                 EnvironmentVariables=[
                     {
@@ -1283,7 +1283,7 @@ def generate_single_account_run_projects(
         TimeoutInMinutes=480,
         Environment=codebuild.Environment(
             ComputeType=t.Ref(deploy_environment_compute_type_parameter),
-            Image="aws/codebuild/standard:4.0",
+            Image=constants.CODEBUILD_DEFAULT_IMAGE,
             Type="LINUX_CONTAINER",
             EnvironmentVariables=[
                 {
@@ -1374,7 +1374,7 @@ def generate_single_action_run_projects(
         TimeoutInMinutes=480,
         Environment=codebuild.Environment(
             ComputeType=t.Ref(deploy_environment_compute_type_parameter),
-            Image="aws/codebuild/standard:4.0",
+            Image=constants.CODEBUILD_DEFAULT_IMAGE,
             Type="LINUX_CONTAINER",
             EnvironmentVariables=[
                 {

servicecatalog_puppet/waluigi/shared_tasks/task_topological_generations_without_scheduler.py

@@ -41,7 +41,9 @@ def get_next_task_to_run(tasks_to_run, resources, all_tasks):
         task_to_run = all_tasks[task_reference_to_run]
         status = task_to_run.get(QUEUE_STATUS, NOT_SET)
         # check if not running or has previously run
-        if status == NOT_SET:
+        if status == IN_PROGRESS:
+            task_permanently_blocked_status.append(False)
+        elif status == NOT_SET:
             is_currently_blocked, is_permanently_blocked = has_dependencies_remaining(
                 task_to_run, all_tasks
             )

servicecatalog_puppet/waluigi/shared_tasks/task_topological_generations_without_scheduler_unit_test.py

@@ -4,6 +4,7 @@
     BLOCKED,
     COMPLETED,
     ERRORED,
+    IN_PROGRESS,
     NOT_SET,
     QUEUE_STATUS,
 )
@@ -375,3 +376,62 @@ def test_get_next_task_to_run_with_blocked_dependencies(self,):
         # verify
         self.assertEqual(expected_should_shutdown, actual_should_shutdown)
         self.assertEqual(expected_next_task, actual_next_task)
+
+    def generate_spoke_execution(self):
+        all_tasks = {
+            "create-policies": {
+                "task_reference": "create-policies",
+                "resources_required": [
+                    "CLOUDFORMATION_CREATE_OR_UPDATE_eu-west-1_OF_XXXX"
+                ],
+                QUEUE_STATUS: COMPLETED,
+            },
+            "generate-manifest": {
+                "task_reference": "generate-manifest",
+                "dependencies_by_reference": ["create-policies"],
+                QUEUE_STATUS: COMPLETED,
+            },
+            "run-deploy-in-spoke_YYYYY": {
+                "task_reference": "run-deploy-in-spoke_YYYYY",
+                "dependencies_by_reference": ["generate-manifest"],
+                QUEUE_STATUS: IN_PROGRESS,
+            },
+        }
+
+        task_to_run = all_tasks.get("run-deploy-in-spoke_YYYYY")
+
+        return task_to_run, all_tasks, list(all_tasks.keys())
+
+    def test_has_dependencies_remaining_with_spoke_execution(self,):
+        # setup
+        (task_to_run, all_tasks, _,) = self.generate_spoke_execution()
+
+        expected_is_currently_blocked = False
+        expected_is_permanently_blocked = False
+
+        # exercise
+        (
+            actual_is_currently_blocked,
+            actual_is_permanently_blocked,
+        ) = self.sut.has_dependencies_remaining(task_to_run, all_tasks)
+
+        # verify
+        self.assertEqual(expected_is_currently_blocked, actual_is_currently_blocked)
+        self.assertEqual(expected_is_permanently_blocked, actual_is_permanently_blocked)
+
+    def test_get_next_task_to_run_with_spoke_execution(self,):
+        # setup
+        (_, all_tasks, tasks_to_run,) = self.generate_spoke_execution()
+        resources = dict()
+
+        expected_next_task = None
+        expected_should_shutdown = False
+
+        # execute
+        actual_next_task, actual_should_shutdown = self.sut.get_next_task_to_run(
+            tasks_to_run, resources, all_tasks
+        )
+
+        # verify
+        self.assertEqual(expected_should_shutdown, actual_should_shutdown)
+        self.assertEqual(expected_next_task, actual_next_task)

servicecatalog_puppet/workflow/generate/generate_policies_task.py

@@ -14,7 +14,7 @@ class GeneratePolicies(tasks.TaskWithReference):
     organizations_to_share_with = luigi.ListParameter()
     ous_to_share_with = luigi.ListParameter()
     accounts_to_share_with = luigi.ListParameter()
-    cachable_level = constants.CACHE_LEVEL_LOW
+    cachable_level = constants.CACHE_LEVEL_NO_CACHE
 
     def params_for_results_display(self):
         return {

servicecatalog_puppet/workflow/launch/run_deploy_in_spoke_task.py

@@ -10,7 +10,7 @@
 class RunDeployInSpokeTask(tasks.TaskWithReference):
     account_id = luigi.Parameter()
     generate_manifest_ref = luigi.Parameter()
-    cachable_level = constants.CACHE_LEVEL_LOW
+    cachable_level = constants.CACHE_LEVEL_NO_CACHE
 
     def params_for_results_display(self):
         return {
@@ -35,5 +35,6 @@ def run(self):
                 environmentVariablesOverride=vars,
                 computeTypeOverride=spoke_execution_mode_deploy_env,
                 buildspecOverride=build_spec,
+                imageOverride=constants.CODEBUILD_DEFAULT_IMAGE,
             )
         self.write_output(dict(account_id=self.account_id, **response))

setup.py

@@ -72,7 +72,7 @@
 
 setup_kwargs = {
     'name': 'aws-service-catalog-puppet',
-    'version': '0.222.0',
+    'version': '0.223.0',
     'description': 'Making it easier to deploy ServiceCatalog products',
     'long_description': '# aws-service-catalog-puppet\n\n![logo](./docs/logo.png) \n\n## Badges\n\n[![codecov](https://codecov.io/gh/awslabs/aws-service-catalog-puppet/branch/master/graph/badge.svg?token=e8M7mdsmy0)](https://codecov.io/gh/awslabs/aws-service-catalog-puppet)\n\n\n## What is it?\nThis is a python3 framework that makes it easier to share multi region AWS Service Catalog portfolios and makes it \npossible to provision products into accounts declaratively using a metadata based rules engine.\n\nWith this framework you define your accounts in a YAML file.  You give each account a set of tags, a default region and \na set of enabled regions.\n\nOnce you have done this you can define portfolios should be shared with each set of accounts using the tags and you \ncan specify which regions the shares occur in.\n\nIn addition to this, you can also define products that should be provisioned into accounts using the same tag based \napproach.  The framework will assume role into the target account and provision the product on your behalf.\n\n\n## Getting started\n\nYou can read the [installation how to](https://service-catalog-tools-workshop.com/30-how-tos/10-installation/30-service-catalog-puppet.html)\nor you can read through the [every day use](https://service-catalog-tools-workshop.com/30-how-tos/50-every-day-use.html)\nguides.\n\nYou can read the [documentation](https://aws-service-catalog-puppet.readthedocs.io/en/latest/) to understand the inner \nworkings. \n\n\n## Going further\n\nThe framework is one of a pair.  The other is [aws-service-catalog-factory](https://github.com/awslabs/aws-service-catalog-factory).\nWith Service Catalog Factory you can create pipelines that deploy multi region portfolios very easily. \n\n## License\n\nThis library is licensed under the Apache 2.0 License. \n \n',
     'author': 'Eamonn Faherty',