Refactored IAM Controller so that roles and policies get their own st…

…ack. This resolves dependency problems.

src/aim/cftemplates/asg.py

@@ -18,10 +18,12 @@ def __init__(self,
                  asg_config,
                  asg_config_ref,
                  role_profile_arn,
-                 ec2_manager_user_data_script):
+                 ec2_manager_user_data_script,
+                 ec2_manager_cache_id ):
 
         #aim_ctx.log("ASG CF Template init")
         self.subenv_ctx = subenv_ctx
+        self.ec2_manager_cache_id = ec2_manager_cache_id
         segment_stack = self.subenv_ctx.get_segment_stack(asg_config.segment)
 
         # Super Init:
@@ -114,6 +116,8 @@ def __init__(self,
 AWSTemplateFormatVersion: '2010-09-09'
 Description: 'ASG: Auto Scaling Group and Launch Configuration'
 
+# EC2 Manager Cache ID: %s
+
 Parameters:
   LCEBSOptimized:
     Description: 'Boolean to toggle Optimized EBS I/O.'
@@ -323,7 +327,7 @@ def __init__(self,
 Outputs:
   ASGName:
     Value: !Ref ASG
-"""
+""" % self.ec2_manager_cache_id
         self.register_stack_output_config(asg_config_ref, 'ASGName')
 
         asg_table = {
@@ -341,6 +345,6 @@ def validate(self):
         #self.aim_ctx.log("Validating ASG Template")
         super().validate()
 
-    def get_outputs_key_from_ref(self, aim_ref):
+    def get_outputs_key_from_ref(self, ref):
         # There is only one output key
         return "ASGName"

src/aim/cftemplates/cftemplates.py

@@ -250,7 +250,7 @@ def aim_sub(self):
                     # Replace the ${}
                     sub_var = self.body[rep_1_idx:rep_1_idx+(rep_2_idx-rep_1_idx)]
                     #print("Sub var: %s" % (sub_var))
-                    self.body = self.body.replace(sub_var, sub_value)
+                    self.body = self.body.replace(sub_var, sub_value, 1)
                 else:
                     #print("break 3")
                     break

src/aim/cftemplates/iam_managed_policies.py

@@ -11,20 +11,17 @@ class IAMManagedPolicies(CFTemplate):
     def __init__(self,
                  aim_ctx,
                  account_ctx,
-                 iam_context_id,
-                 policies_by_order):
+                 policy_context,
+                 template_name):
         #aim_ctx.log("IAMManagedPolicies CF Template init")
-        aws_name = "Policy"
+        aws_name = '-'.join([template_name, "Policy"])
 
         super().__init__(aim_ctx,
                          account_ctx,
                          config_ref="",
                          aws_name=aws_name,
                          iam_capabilities=["CAPABILITY_NAMED_IAM"])
-
-        self.policies_by_order = policies_by_order
-        self.iam_context_id = iam_context_id
-
+        self.policy_context = policy_context
         # Define the Template
         template_fmt = """
 AWSTemplateFormatVersion: '2010-09-09'
@@ -88,33 +85,31 @@ def __init__(self,
      Type: {0[type]:s}
      Description: {0[description]:s}
 """
-        managed_policies_yaml = ""
         policy_table.clear()
-        for policy_context in self.policies_by_order:
-            if policy_context['template_params']:
-                for param_table in policy_context['template_params']:
-                    self.set_parameter(param_table['key'], param_table['value'])
-                    parameters_yaml += parameter_fmt.format(param_table)
-
-            policy_config = policy_context['config']
-            policy_id = policy_context['id']
-            # Name
-            policy_table['name'] = self.gen_policy_name(policy_id)
-            policy_table['cf_resource_name_prefix'] = self.get_cf_resource_name_prefix(policy_id)
-
-            # Roles
-            policy_table['roles'] = ""
-            for role in policy_config.roles:
-                policy_table['roles'] += role_fmt % (role)
-            # Path
-            policy_table['path'] = policy_config.path
-            # Statement
-            policy_table['statement'] = self.gen_statement_yaml(policy_config.statement)
-            # Initialize Parameters
-            # Resources
-            resources_yaml += policy_fmt.format(policy_table)
-            # Outputs
-            outputs_yaml += policy_outputs_fmt.format(policy_table)
+        if policy_context['template_params']:
+            for param_table in policy_context['template_params']:
+                self.set_parameter(param_table['key'], param_table['value'])
+                parameters_yaml += parameter_fmt.format(param_table)
+
+        policy_config = policy_context['config']
+        policy_id = policy_context['id']
+        # Name
+        policy_table['name'] = self.gen_policy_name(policy_id)
+        policy_table['cf_resource_name_prefix'] = self.get_cf_resource_name_prefix(policy_id)
+
+        # Roles
+        policy_table['roles'] = ""
+        for role in policy_config.roles:
+            policy_table['roles'] += role_fmt % (role)
+        # Path
+        policy_table['path'] = policy_config.path
+        # Statement
+        policy_table['statement'] = self.gen_statement_yaml(policy_config.statement)
+        # Initialize Parameters
+        # Resources
+        resources_yaml += policy_fmt.format(policy_table)
+        # Outputs
+        outputs_yaml += policy_outputs_fmt.format(policy_table)
 
         template_table['parameters_yaml'] = ""
         if parameters_yaml != "":
@@ -131,7 +126,8 @@ def validate(self):
 
     # Generate a name valid in CloudFormation
     def gen_policy_name(self, policy_id):
-        policy_name = '-'.join([self.iam_context_id, policy_id])
+        policy_context_hash = self.aim_ctx.md5sum(str_data=self.policy_context['ref'])[:8].upper()
+        policy_name = '-'.join([policy_context_hash, policy_id])
         policy_name = self.aim_ctx.normalize_name(policy_name, '-', False)
         return policy_name
 
@@ -143,9 +139,9 @@ def get_cf_resource_name_prefix(self, resource_name):
     # be consolidated in cftemplates.py...
     def gen_statement_yaml(self, statements):
         statement_fmt = """
-              - Effect: {0[effect]:s}
-                Action:{0[action_list]:s}
-                Resource:{0[resource_list]:s}
+          - Effect: {0[effect]:s}
+            Action:{0[action_list]:s}
+            Resource:{0[resource_list]:s}
 """
         statement_table = {
             'effect': None,
@@ -154,9 +150,9 @@ def gen_statement_yaml(self, statements):
         }
 
         quoted_list_fmt = """
-                  - '{0}'"""
+              - '{0}'"""
         unquoted_list_fmt = """
-                  - {0}"""
+              - {0}"""
         statement_yaml = ""
         for statement in statements:
             statement_table['effect'] = ""

src/aim/cftemplates/iam_roles.py

@@ -11,19 +11,20 @@ class IAMRoles(CFTemplate):
     def __init__(self,
                  aim_ctx,
                  account_ctx,
-                 iam_context_id,
-                 roles_by_order):
+                 template_name,
+                 role_ref,
+                 role_id,
+                 role_config,
+                 template_params):
         #aim_ctx.log("IAMRoles CF Template init")
-        aws_name = "Roles"
-
+        aws_name = '-'.join([template_name, "Role"])
         super().__init__(aim_ctx,
                          account_ctx,
                          config_ref="",
                          aws_name=aws_name,
                          iam_capabilities=["CAPABILITY_NAMED_IAM"])
 
-        self.roles_by_order = roles_by_order
-        self.iam_context_id = iam_context_id
+        self.role_ref = role_ref
 
         # Define the Template
         template_fmt = """
@@ -107,62 +108,57 @@ def __init__(self,
      Type: {0[type]:s}
      Description: {0[description]:s}
 """
-        roles_yaml = ""
         iam_role_table.clear()
-        for role_context in self.roles_by_order:
-            if role_context['template_params']:
-                for param_table in role_context['template_params']:
-                    self.set_parameter(param_table['key'], param_table['value'])
-                    parameters_yaml += parameter_fmt.format(param_table)
-
-            role_config = role_context['config']
-            role_id = role_context['id']
-
-            # Role
-            role_path_param_name = self.get_cf_resource_name_prefix(role_id) + "RolePath"
-            iam_role_table['role_path_param_name'] = role_path_param_name
-            iam_role_table['role_name'] = self.gen_iam_role_name("Role", role_id)
-            iam_role_table['cf_resource_name_prefix'] = self.get_cf_resource_name_prefix(role_id)
-
-            # Assume Role Principal
-            principal_yaml = ""
-            if role_config.assume_role_policy != None:
-                if len(role_config.assume_role_policy.service) > 0:
-                    principal_yaml += """
+        if template_params:
+            for param_table in template_params:
+                self.set_parameter(param_table['key'], param_table['value'])
+                parameters_yaml += parameter_fmt.format(param_table)
+
+        # Role
+        role_path_param_name = self.get_cf_resource_name_prefix(role_id) + "RolePath"
+        iam_role_table['role_path_param_name'] = role_path_param_name
+        iam_role_table['role_name'] = self.gen_iam_role_name("Role", role_id)
+        iam_role_table['cf_resource_name_prefix'] = self.get_cf_resource_name_prefix(role_id)
+
+        # Assume Role Principal
+        principal_yaml = ""
+        if role_config.assume_role_policy != None:
+            if len(role_config.assume_role_policy.service) > 0:
+                principal_yaml += """
               Service:"""
-                    for service_item in role_config.assume_role_policy.service:
-                        principal_yaml += """
-                - """ + service_item
-                elif role_config.assume_role_policy.aws != '':
+                for service_item in role_config.assume_role_policy.service:
                     principal_yaml += """
+                - """ + service_item
+            elif role_config.assume_role_policy.aws != '':
+                principal_yaml += """
               AWS:"""
-                    for aws_item in role_config.assume_role_policy.aws:
-                        principal_yaml += """
+                for aws_item in role_config.assume_role_policy.aws:
+                    principal_yaml += """
                 - """ + aws_item
-            else:
-                pass
-            iam_role_table['assume_role_principal'] = principal_yaml
-
-            if role_config.policies:
-                iam_role_table['inline_policies'] = self.gen_role_policies(role_config.policies)
-            else:
-                iam_role_table['inline_policies'] = ""
-
-            # Instance Profile
-            if role_config.instance_profile == True:
-                iam_role_table['profile_name'] = self.gen_iam_role_name("Profile", role_id)
-                iam_role_table['instance_profile'] = iam_profile_fmt.format(iam_role_table)
-            else:
-                iam_role_table['instance_profile'] = ""
-
-            parameters_yaml += iam_role_params_fmt.format(iam_role_table)
-            resources_yaml += iam_role_fmt.format(iam_role_table)
-            outputs_yaml += iam_role_outputs_fmt.format(iam_role_table)
-
-            if role_config.instance_profile == True:
-                outputs_yaml += iam_profile_outputs_fmt.format(iam_role_table)
-            # Initialize Parameters
-            self.set_parameter(role_path_param_name, role_config.path)
+        else:
+            pass
+        iam_role_table['assume_role_principal'] = principal_yaml
+
+        if role_config.policies:
+            iam_role_table['inline_policies'] = self.gen_role_policies(role_config.policies)
+        else:
+            iam_role_table['inline_policies'] = ""
+
+        # Instance Profile
+        if role_config.instance_profile == True:
+            iam_role_table['profile_name'] = self.gen_iam_role_name("Profile", role_id)
+            iam_role_table['instance_profile'] = iam_profile_fmt.format(iam_role_table)
+        else:
+            iam_role_table['instance_profile'] = ""
+
+        parameters_yaml += iam_role_params_fmt.format(iam_role_table)
+        resources_yaml += iam_role_fmt.format(iam_role_table)
+        outputs_yaml += iam_role_outputs_fmt.format(iam_role_table)
+
+        if role_config.instance_profile == True:
+            outputs_yaml += iam_profile_outputs_fmt.format(iam_role_table)
+        # Initialize Parameters
+        self.set_parameter(role_path_param_name, role_config.path)
 
         template_table['parameters_yaml'] = parameters_yaml
         template_table['resources_yaml'] = resources_yaml
@@ -175,7 +171,7 @@ def validate(self):
 
     # Generate a name valid in CloudFormation
     def gen_iam_role_name(self, role_type, role_id):
-        iam_context_hash = self.aim_ctx.md5sum(str_data=self.iam_context_id)[:8].upper()
+        iam_context_hash = self.aim_ctx.md5sum(str_data=self.role_ref)[:8].upper()
         role_name = '-'.join([iam_context_hash, role_type[0], role_id])
         role_name = self.aim_ctx.normalize_name(role_name, '-', False)
         return role_name

src/aim/cftemplates/s3.py

@@ -121,18 +121,16 @@ def __init__(self,
 
         # Bucket Policy
         if len(bucket_config.policy) > 0:
-            if not bucket_policy_only:
+            if bucket_policy_only == False:
                 s3_policy_table['cf_policy_depends_on'] = s3_policy_depends_on_fmt.format(s3_bucket_table)
 
             s3_policy_table['cf_resource_name_prefix'] = s3_bucket_table['cf_resource_name_prefix']
             s3_policy_table['bucket_name'] = s3_bucket_table['bucket_name']
             s3_policy_table['policy_statements'] = ""
             # Statement
             for policy_statement in bucket_config.policy:
-                if policy_statement.written_to_template == True:
+                if policy_statement.processed == True:
                     continue
-                else:
-                    policy_statement.written_to_template = True
                 s3_policy_statement_table['action_list'] = ""
                 s3_policy_statement_table['principal'] = ""
                 s3_policy_statement_table['effect'] = ""
@@ -164,6 +162,7 @@ def __init__(self,
         template_table['parameters_yaml'] = parameters_yaml
         template_table['resources_yaml'] = resources_yaml
         template_table['outputs_yaml'] = outputs_yaml
+
         self.set_template(template_fmt.format(template_table))
 
     def validate(self):
@@ -183,5 +182,5 @@ def get_outputs_key_from_ref(self, aim_ref):
 
     def delete(self):
         s3_ctl = self.aim_ctx.get_controller('S3')
-        s3_ctl.empty_bucket(self.s3_context_id, self.bucket_context['ref'])
+        s3_ctl.empty_bucket(self.bucket_context['ref'])
         super().delete()

src/aim/commands/cli.py

@@ -5,6 +5,7 @@
 from aim.commands.cmd_delete import delete_command
 from aim.commands.cmd_describe import describe_command
 from aim.commands.cmd_validate import validate_command
+from aim.commands.cmd_shell import shell_command
 from aim.commands.helpers import pass_aim_context
 
 @click.group()
@@ -26,3 +27,4 @@ def cli(ctx, verbose):
 cli.add_command(delete_command)
 cli.add_command(describe_command)
 cli.add_command(validate_command)
+cli.add_command(shell_command)

src/aim/commands/cmd_shell.py

@@ -0,0 +1,29 @@
+import click
+import sys
+from aim.core.exception import StackException
+from aim.commands.helpers import pass_aim_context, controller_args, aim_home_option, init_aim_home_option
+
+@click.command('shell', short_help='Open a shell to an instance.')
+@click.argument("netenv_id", required=True, type=click.STRING)
+@click.argument("env_id", required=True, type=click.STRING)
+@click.argument("region", required=True, type=click.STRING)
+@click.argument("instance_ref", required=True, type=click.STRING)
+@aim_home_option
+@pass_aim_context
+def shell_command(aim_ctx, netenv_id, env_id, region, instance_ref, home='.'):
+    """Open a shell to an instance"""
+
+    init_aim_home_option(aim_ctx, home)
+    if not aim_ctx.home:
+        print('AIM configuration directory needs to be specified with either --home or AIM_HOME environment variable.')
+        sys.exit()
+
+    aim_ctx.init_project()
+    config_arg = {
+            'netenv_id': netenv_id,
+            'subenv_id': env_id,
+            'region' : region,
+        }
+    aim_ctx.get_controller('NetEnv', config_arg)
+    full_ref = 'netenv.ref %s.subenv.%s.%s.%s' % (netenv_id, env_id, region, instance_ref)
+    aim_ctx.get_ref(full_ref)

src/aim/commands/helpers.py

@@ -4,6 +4,7 @@
 
 pass_aim_context = click.make_pass_decorator(AimContext, ensure=True)
 
+
 def controller_args(func):
     """
     decorator to add controller args