Release/0.0.42 (#48)

Now using a central SQS queue to store Cloudformation stack events from each region.

.gitignore

@@ -6,3 +6,4 @@ aws_service_catalog_puppet.egg-info/
 dist
 __pycache__
 output
+.envrc

docs/source/puppet/notifications.md

@@ -5,9 +5,14 @@ You can listen to the AWS CloudFormation stack events from your product provisio
 
 This is the recommended way of discovering provisioning errors.
 
-When you bootstraped your account you will have created an AWS SNS Topic: 
+When you bootstraped your account you will have created an AWS SQS Queue: 
 ```servicecatalog-puppet-cloudformation-events``` in your default region.
 
+You will also have SNS Topics in each region configured to push events to this queue:
+```servicecatalog-puppet-cloudformation-regional-events```
+
 Please note this will only receive notifications for products provisioned using 
 ServiceCatalog-Puppet - any self service vending from AWS Service Catalog will not 
-use this topic.
+publish to this queue.
+
+You should handle consuming the queue and have your own error handling code.

servicecatalog_puppet/asset_helpers.py

@@ -12,4 +12,4 @@ def read_from_site_packages(what):
     return open(
         resolve_from_site_packages(what),
         'r'
-    ).read()
+    ).read()

servicecatalog_puppet/cli.py

@@ -7,13 +7,15 @@
 import yaml
 import logging
 import os
+
+from jinja2 import Template
 from pykwalify.core import Core
 
 from betterboto import client as betterboto_client
 
 from servicecatalog_puppet.commands.list_launches import do_list_launches
 from servicecatalog_puppet.utils import manifest as manifest_utils
-from servicecatalog_puppet.asset_helpers import resolve_from_site_packages
+from servicecatalog_puppet.asset_helpers import resolve_from_site_packages, read_from_site_packages
 from servicecatalog_puppet.constants import CONFIG_PARAM_NAME, CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN
 from servicecatalog_puppet.core import get_org_iam_role_arn, write_templates, create_share_template, \
     deploy_launches
@@ -89,8 +91,8 @@ def bootstrap_spoke_as(puppet_account_id, iam_role_arns):
         index += 1
 
     with betterboto_client.CrossMultipleAccountsClientContextManager(
-        'cloudformation',
-        cross_accounts
+            'cloudformation',
+            cross_accounts
     ) as cloudformation:
         do_bootstrap_spoke(puppet_account_id, cloudformation, get_puppet_version())
 
@@ -225,13 +227,78 @@ def set_org_iam_role_arn(org_iam_role_arn):
 @click.argument('puppet_account_id')
 def bootstrap_org_master(puppet_account_id):
     with betterboto_client.ClientContextManager(
-        'cloudformation',
+            'cloudformation',
     ) as cloudformation:
         org_iam_role_arn = do_bootstrap_org_master(
             puppet_account_id, cloudformation, get_puppet_version()
         )
     click.echo("Bootstrapped org master, org-iam-role-arn: {}".format(org_iam_role_arn))
 
 
+@cli.command()
+def quick_start():
+    click.echo("Quick Start running...")
+    puppet_version = get_puppet_version()
+    with betterboto_client.ClientContextManager('sts') as sts:
+        puppet_account_id = sts.get_caller_identity().get('Account')
+        click.echo("Going to use puppet_account_id: {}".format(puppet_account_id))
+    click.echo("Bootstrapping account as a spoke")
+    with betterboto_client.ClientContextManager('cloudformation') as cloudformation:
+        do_bootstrap_spoke(puppet_account_id, cloudformation, puppet_version)
+
+    click.echo("Setting the config")
+    content = yaml.safe_dump({
+        "regions": [
+            'eu-west-1',
+            'eu-west-2',
+            'eu-west-3'
+        ]
+    })
+    with betterboto_client.ClientContextManager('ssm') as ssm:
+        ssm.put_parameter(
+            Name=CONFIG_PARAM_NAME,
+            Type='String',
+            Value=content,
+            Overwrite=True,
+        )
+        click.echo("Bootstrapping account as the master")
+        org_iam_role_arn = do_bootstrap_org_master(
+            puppet_account_id, cloudformation, puppet_version
+        )
+        ssm.put_parameter(
+            Name=CONFIG_PARAM_NAME_ORG_IAM_ROLE_ARN,
+            Type='String',
+            Value=org_iam_role_arn,
+            Overwrite=True,
+        )
+    click.echo("Bootstrapping the account now!")
+    do_bootstrap(puppet_version)
+
+    if os.path.exists('ServiceCatalogPuppet'):
+        click.echo("Found ServiceCatalogPuppet so not cloning or seeding")
+    else:
+        click.echo("Cloning for you")
+        command = "git clone " \
+                  "--config 'credential.helper=!aws codecommit credential-helper $@' " \
+                  "--config 'credential.UseHttpPath=true' " \
+                  "https://git-codecommit.{}.amazonaws.com/v1/repos/ServiceCatalogPuppet".format(
+            os.environ.get("AWS_DEFAULT_REGION")
+        )
+        os.system(command)
+        click.echo("Seeding")
+        manifest = Template(
+            read_from_site_packages(os.path.sep.join(["manifests","manifest-quickstart.yaml"]))
+        ).render(
+            ACCOUNT_ID=puppet_account_id
+        )
+        open(os.path.sep.join(["ServiceCatalogPuppet", "manifest.yaml"]), 'w').write(
+            manifest
+        )
+        click.echo("Pushing manifest")
+        os.system("cd ServiceCatalogPuppet && git add manifest.yaml && git commit -am 'initial add' && git push")
+
+    click.echo("All done!")
+
+
 if __name__ == "__main__":
     cli()

servicecatalog_puppet/core.py

@@ -205,13 +205,15 @@ def write_share_template(portfolio_use_by_account, region, host_account_id, shar
 
 
 def create_share_template(deployment_map, puppet_account_id):
+    logger.info("deployment_map: {}".format(deployment_map))
     ALL_REGIONS = get_regions()
     for region in ALL_REGIONS:
         logger.info("starting to build shares for region: {}".format(region))
         with betterboto_client.ClientContextManager('servicecatalog', region_name=region) as servicecatalog:
             portfolio_ids = {}
             args = {}
             while True:
+
                 response = servicecatalog.list_portfolios(
                     **args
                 )
@@ -231,7 +233,7 @@ def create_share_template(deployment_map, puppet_account_id):
                 if portfolio_use_by_account.get(account_id) is None:
                     portfolio_use_by_account[account_id] = []
                 for launch_id, launch in launch_details.get('launches').items():
-                    logger.info(portfolio_ids)
+                    logger.info("portfolio ids: {}".format(portfolio_ids))
                     p = portfolio_ids[launch.get('portfolio')]
                     if p not in portfolio_use_by_account[account_id]:
                         portfolio_use_by_account[account_id].append(p)
@@ -358,6 +360,11 @@ def deploy_launch_to_account_and_region(
     launch_name = launch.get('launch_name')
     stack_name = "-".join([PREFIX, account, region, launch_name])
     logger.info('Creating plan, params: {}'.format(params))
+    regional_sns_topic = "arn:aws:sns:{}:{}:servicecatalog-puppet-cloudformation-regional-events".format(
+        region,
+        puppet_account_id
+    )
+    logger.info("regional_sns_topic is: {}".format(regional_sns_topic))
     response = service_catalog.create_provisioned_product_plan(
         PlanName=stack_name,
         PlanType='CLOUDFORMATION',
@@ -377,10 +384,7 @@ def deploy_launch_to_account_and_region(
             },
         ],
         NotificationArns=[
-            "arn:aws:sns:{}:{}:servicecatalog-puppet-cloudformation-events".format(
-                get_home_region(),
-                puppet_account_id
-            ),
+            regional_sns_topic,
         ],
     )
     logger.info('Plan created, waiting for completion')

servicecatalog_puppet/manifests/manifest-quickstart.yaml

@@ -0,0 +1,57 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+schema: puppet-2019-04-01
+
+accounts:
+  - account_id: '{{ ACCOUNT_ID }}'
+    name: '{{ ACCOUNT_ID }}'
+    default_region: eu-west-1
+    regions_enabled:
+      - eu-west-1
+      - eu-west-2
+      - eu-west-3
+    tags:
+      - type:prod
+      - partition:eu
+      - scope:pci
+
+launches:
+  devops-iam-for-prod:
+    portfolio: demo-central-it-team-portfolio
+    product: account-iam
+    version: v1
+    parameters:
+      RoleName:
+        default: DevOps
+      Path:
+        default: /human-roles/
+    deploy_to:
+      tags:
+        - tag: type:prod
+          regions: default_region
+  sysops-iam-for-prod:
+    portfolio: demo-central-it-team-portfolio
+    product: account-iam
+    version: v1
+    parameters:
+      RoleName:
+        default: SysOps
+      Path:
+        default: /human-roles/
+    deploy_to:
+      tags:
+        - tag: type:prod
+          regions: eu-west-2
+  secops-iam-for-prod:
+    portfolio: demo-central-it-team-portfolio
+    product: account-iam
+    version: v1
+    parameters:
+      RoleName:
+        default: SecOps
+      Path:
+        default: /human-roles/
+    deploy_to:
+      tags:
+        - tag: type:prod
+          regions: eu-west-3

servicecatalog_puppet/manifests/manifest-simple.yaml

@@ -8,7 +8,7 @@ accounts:
     default_region: eu-west-1
     regions_enabled:
       - eu-west-1
-      - eu-west-1
+      - eu-west-2
     tags:
       - type:prod
       - partition:eu

servicecatalog_puppet/servicecatalog-puppet-regional.template.yaml

@@ -36,6 +36,19 @@ Resources:
       VersioningConfiguration:
         Status: Enabled
 
+  RegionalProductTopic:
+    Type: AWS::SNS::Topic
+    Properties:
+      DisplayName: servicecatalog-puppet-cloudformation-regional-events
+      TopicName: servicecatalog-puppet-cloudformation-regional-events
+      Subscription:
+        - Endpoint: !Sub "arn:aws:sqs:${DefaultRegionValue}:${AWS::AccountId}:servicecatalog-puppet-cloudformation-events"
+          Protocol: "sqs"
+
+
+
 Outputs:
   Version:
     Value: !GetAtt Param.Value
+  RegionalProductTopic:
+    Value: !Ref RegionalProductTopic

servicecatalog_puppet/servicecatalog-puppet-spoke.template.yaml

@@ -76,4 +76,4 @@ Resources:
 
 Outputs:
   Version:
-    Value: !GetAtt Param.Value
+    Value: !GetAtt Param.Value

servicecatalog_puppet/servicecatalog-puppet.template.yaml

@@ -368,17 +368,33 @@ Resources:
                 - aws codepipeline start-pipeline-execution  --name servicecatalog-puppet-pipeline
       TimeoutInMinutes: 30
 
+  CloudFormationEventsQueue:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: servicecatalog-puppet-cloudformation-events
 
-  ProductTopic:
-    Type: AWS::SNS::Topic
+  CloudFormationEventsQueuePolicy:
+    Type: AWS::SQS::QueuePolicy
     Properties:
-      DisplayName: servicecatalog-puppet-cloudformation-events
-      TopicName: servicecatalog-puppet-cloudformation-events
+      PolicyDocument:
+        Id: AllowSNS
+        Version: '2012-10-17'
+        Statement:
+          - Sid: allow-send-message
+            Effect: Allow
+            Principal: "*"
+            Action:
+              - sqs:SendMessage
+            Resource: "*"
+            Condition:
+              ArnEquals:
+                aws:SourceArn: !Sub "arn:aws:sns:*:${AWS::AccountId}:servicecatalog-puppet-cloudformation-regional-events"
+      Queues:
+        - !Ref CloudFormationEventsQueue
+
 
 Outputs:
-  ProductTopicArn:
-    Value: !Ref ProductTopic
-  ProductTopicName:
-    Value: !GetAtt ProductTopic.TopicName
+  CloudFormationEventsQueueArn:
+    Value: !GetAtt CloudFormationEventsQueue.Arn
   Version:
-    Value: !GetAtt Param.Value
+    Value: !GetAtt Param.Value

setup.py

@@ -11,7 +11,7 @@
 
 setuptools.setup(
     name="aws-service-catalog-puppet",
-    version="0.0.41",
+    version="0.0.42",
     author="Eamonn Faherty",
     author_email="aws-service-catalog-tools@amazon.com",
     description="Making it easier to deploy ServiceCatalog products",