sharing an s3 client to download from the cache instead of doing per … (

#644)

* sharing an s3 client to download from the cache instead of doing per time
* bumping ubuntu version for github action

.github/workflows/workflow.yml

@@ -7,7 +7,7 @@ jobs:
       matrix:
         python-version: [3.9]
         poetry-version: [1.2.0]
-        os: [ubuntu-18.04]
+        os: [ubuntu-20.04]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2

pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "aws-service-catalog-puppet"
-version = "0.224.0"
+version = "0.225.0"
 description = "Making it easier to deploy ServiceCatalog products"
 classifiers = ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Natural Language :: English"]
 homepage = "https://service-catalog-tools-workshop.com/"

servicecatalog_puppet/config.py

@@ -282,6 +282,10 @@ def get_reporting_role_arn(puppet_account_id):
     return get_role_arn(puppet_account_id, constants.REPORTING_ROLE_NAME)
 
 
+def get_cache_download_role_arn(puppet_account_id):
+    return get_role_arn(puppet_account_id, constants.CACHE_DOWNLOADING_ROLE_NAME)
+
+
 def get_spoke_scheduler_threads_or_processes():
     return os.environ.get(
         environmental_variables.SPOKE_SCHEDULER_THREADS_OR_PROCESSES,

servicecatalog_puppet/constants.py

@@ -382,6 +382,7 @@
 
 
 REPORTING_ROLE_NAME = "PuppetRoleForReporting"
+CACHE_DOWNLOADING_ROLE_NAME = "PuppetRoleForDownloadingFromCache"
 
 SHARE_PRINCIPALS_DEFAULT = False
 

servicecatalog_puppet/template_builder/hub/bootstrap.py

@@ -431,6 +431,44 @@ def get_template(
         )
     )
 
+    template.add_resource(
+        iam.Role(
+            constants.CACHE_DOWNLOADING_ROLE_NAME,
+            RoleName=constants.CACHE_DOWNLOADING_ROLE_NAME,
+            MaxSessionDuration=43200,
+            AssumeRolePolicyDocument={
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Action": ["sts:AssumeRole"],
+                        "Effect": "Allow",
+                        "Principal": {
+                            "AWS": {
+                                "Fn::Sub": "arn:${AWS::Partition}:iam::${AWS::AccountId}:root"
+                            }
+                        },
+                    },
+                ],
+            },
+            Policies=[
+                iam.Policy(
+                    PolicyName="DownloadingCacheActions",
+                    PolicyDocument={
+                        "Version": "2012-10-17",
+                        "Statement": [
+                            {
+                                "Action": ["s3:GetObject"],
+                                "Resource": t.Sub("${CachingBucket.Arn}/*"),
+                                "Effect": "Allow",
+                            },
+                        ],
+                    },
+                )
+            ],
+            Path=t.Ref(puppet_role_path_template_parameter),
+        )
+    )
+
     template.add_resource(
         sns.Topic(
             "DryRunNotificationTopic",

servicecatalog_puppet/waluigi/tasks.py

@@ -1,13 +1,16 @@
 #  Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #  SPDX-License-Identifier: Apache-2.0
+import functools
 import json
 import logging
 import os
 import traceback
 from datetime import datetime
 from pathlib import Path
 
-from servicecatalog_puppet import constants
+from betterboto import client as betterboto_client
+
+from servicecatalog_puppet import config, constants
 
 
 logger = logging.getLogger(constants.PUPPET_LOGGER_NAME)
@@ -39,6 +42,15 @@ def record_event(event_type, task, extra_event_data=None):
         f.write(json.dumps(event, default=str, indent=4,))
 
 
+@functools.lru_cache(maxsize=10)
+def get_cache_download_client():
+    return betterboto_client.CrossAccountClientContextManager(
+        "s3",
+        config.get_cache_download_role_arn(config.get_executor_account_id()),
+        "s3-client",
+    )
+
+
 class WaluigiTaskMixin:
     def execute(self):
         if self.should_use_caching:
@@ -55,7 +67,7 @@ def execute(self):
                 if not os.path.exists(target_dir):
                     os.makedirs(target_dir)
                 if not os.path.exists(target):
-                    with self.hub_client("s3") as s3:
+                    with get_cache_download_client() as s3:
                         s3.download_file(Bucket=bucket, Key=key, Filename=target)
                 if not os.path.exists(target):
                     raise Exception(f"{target} was not downloaded from the cache")

setup.py

@@ -72,7 +72,7 @@
 
 setup_kwargs = {
     'name': 'aws-service-catalog-puppet',
-    'version': '0.224.0',
+    'version': '0.225.0',
     'description': 'Making it easier to deploy ServiceCatalog products',
     'long_description': '# aws-service-catalog-puppet\n\n![logo](./docs/logo.png) \n\n## Badges\n\n[![codecov](https://codecov.io/gh/awslabs/aws-service-catalog-puppet/branch/master/graph/badge.svg?token=e8M7mdsmy0)](https://codecov.io/gh/awslabs/aws-service-catalog-puppet)\n\n\n## What is it?\nThis is a python3 framework that makes it easier to share multi region AWS Service Catalog portfolios and makes it \npossible to provision products into accounts declaratively using a metadata based rules engine.\n\nWith this framework you define your accounts in a YAML file.  You give each account a set of tags, a default region and \na set of enabled regions.\n\nOnce you have done this you can define portfolios should be shared with each set of accounts using the tags and you \ncan specify which regions the shares occur in.\n\nIn addition to this, you can also define products that should be provisioned into accounts using the same tag based \napproach.  The framework will assume role into the target account and provision the product on your behalf.\n\n\n## Getting started\n\nYou can read the [installation how to](https://service-catalog-tools-workshop.com/30-how-tos/10-installation/30-service-catalog-puppet.html)\nor you can read through the [every day use](https://service-catalog-tools-workshop.com/30-how-tos/50-every-day-use.html)\nguides.\n\nYou can read the [documentation](https://aws-service-catalog-puppet.readthedocs.io/en/latest/) to understand the inner \nworkings. \n\n\n## Going further\n\nThe framework is one of a pair.  The other is [aws-service-catalog-factory](https://github.com/awslabs/aws-service-catalog-factory).\nWith Service Catalog Factory you can create pipelines that deploy multi region portfolios very easily. \n\n## License\n\nThis library is licensed under the Apache 2.0 License. \n \n',
     'author': 'Eamonn Faherty',