Skip to content

Commit

Permalink
boto3 parameters, ssm parameter and outputs dependencies have been re…
Browse files Browse the repository at this point in the history 
…written
  • Loading branch information
@eamonnfaherty
eamonnfaherty committed Jun 13, 2023
2 parents 8f2fcad + fa6b79d commit 830c39a
Show file tree
Hide file tree
Showing 10 changed files with 275 additions and 188 deletions.
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

[tool.poetry]
name = "aws-service-catalog-puppet"
version = "0.230.1"
version = "0.231.0"
description = "Making it easier to deploy ServiceCatalog products"
classifiers = ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Natural Language :: English"]
homepage = "https://service-catalog-tools-workshop.com/"
Expand Down
116 changes: 81 additions & 35 deletions servicecatalog_puppet/commands/task_reference_helpers/complete_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
from servicecatalog_puppet.commands.task_reference_helpers.generators.ssm_parameter_handler import (
ssm_parameter_handler,
)
from servicecatalog_puppet.waluigi.shared_tasks.task_topological_generations_without_scheduler_unit_test import (
dependency_task_reference,
)
from servicecatalog_puppet.workflow import workflow_utils
from servicecatalog_puppet.workflow.dependencies import resources_factory

Expand Down Expand Up @@ -163,46 +166,43 @@ def generate(puppet_account_id, manifest, output_file_path):
manifest,
)

#
# Second pass - adding get parameters
#
new_tasks = dict()
for task_reference, task in all_tasks.items():
parameters = {}
launch_parameters = (
manifest.get(task.get("section_name"), {})
.get(task.get("item_name"), {})
.get("parameters", {})
)
manifest_parameters = copy.deepcopy(manifest.get("parameters"))
account_parameters = manifest.get_parameters_for_account(task.get("account_id"))

always_merger.merge(parameters, manifest_parameters)
always_merger.merge(parameters, launch_parameters)
always_merger.merge(parameters, account_parameters)

if task.get("status") != constants.TERMINATED:
for parameter_name, parameter_details in parameters.items():
ssm_parameter_handler(
all_tasks,
default_region,
new_tasks,
parameter_details,
puppet_account_id,
task,
task = task_to_add # TODO rename
new_tasks = all_tasks # TODO rename
parameters = {}
launch_parameters = (
manifest.get(task.get("section_name"), {})
.get(task.get("item_name"), {})
.get("parameters", {})
)
boto3_parameter_handler(
new_tasks,
parameter_details,
parameter_name,
puppet_account_id,
task,
manifest_parameters = copy.deepcopy(manifest.get("parameters"))
account_parameters = manifest.get_parameters_for_account(
task.get("account_id")
)

all_tasks.update(new_tasks)
always_merger.merge(parameters, manifest_parameters)
always_merger.merge(parameters, launch_parameters)
always_merger.merge(parameters, account_parameters)

if task.get("status") != constants.TERMINATED:
for parameter_name, parameter_details in parameters.items():
ssm_parameter_handler(
all_tasks,
default_region,
new_tasks,
parameter_details,
puppet_account_id,
task,
)
boto3_parameter_handler(
new_tasks,
parameter_details,
parameter_name,
puppet_account_id,
task,
)

#
# Third pass - replacing dependencies with dependencies_by_reference and adding resources
# Second pass - replacing dependencies with dependencies_by_reference and adding resources
#
for task_reference, task in all_tasks.items():
for dependency in task.get("dependencies", []):
Expand Down Expand Up @@ -263,6 +263,52 @@ def generate(puppet_account_id, manifest, output_file_path):
)
task["resources_required"] = resources

#
# Third pass - setting dependencies between parameters and outputs
#
for task_reference, task in all_tasks.items():
ssm_parameters = task.get("ssm_parameters_tasks_references", {}).items()
dependencies_to_add = []
for parameter_name, parameter_task_reference in ssm_parameters:
for dependency in task.get("dependencies_by_reference"):
ssm_outputs = (
all_tasks.get(dependency)
.get("ssm_outputs_tasks_references", {})
.items()
)
for output_name, output_task_reference in ssm_outputs:
if (
output_task_reference.replace(
constants.SSM_OUTPUTS, constants.SSM_PARAMETERS
)
== parameter_task_reference
):
dependencies_to_add.append(output_task_reference)
all_tasks[parameter_task_reference][
"dependencies_by_reference"
].append(output_task_reference)
task["dependencies_by_reference"].extend(dependencies_to_add)

boto3_parameters = task.get("boto3_parameters_tasks_references", {}).items()
dependencies_to_add = []
for parameter_name, parameter_task_reference in boto3_parameters:
parameter_task = all_tasks.get(parameter_task_reference)
for dependency_task_reference in task.get("dependencies_by_reference"):
if dependency_task_reference not in parameter_task.get(
"dependencies_by_reference"
):
if dependency_task_reference != parameter_task_reference:
if (
all_tasks[dependency_task_reference]["section_name"]
in constants.ALL_SECTION_NAMES
):
parameter_task["dependencies_by_reference"].append(
dependency_task_reference
)

if task_reference in parameter_task["dependencies_by_reference"]:
parameter_task["dependencies_by_reference"].remove(task_reference)

reference = dict(all_tasks=all_tasks,)
workflow_utils.ensure_no_cyclic_dependencies("complete task reference", all_tasks)
open(output_file_path, "w").write(serialisation_utils.dump_as_json(reference))
Expand Down
53 changes: 45 additions & 8 deletions servicecatalog_puppet/commands/task_reference_helpers/generators/boto3_parameter_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,39 @@
from servicecatalog_puppet import constants


def generate_task_reference(
parameter_details, account_id_to_use_for_boto3_call, region_to_use_for_boto3_call,
):
if parameter_details.get("cloudformation_stack_output"):
return (
f"{constants.BOTO3_PARAMETERS}"
f"-cloudformation_stack_output"
f"-{parameter_details.get('cloudformation_stack_output').get('stack_name')}"
f"-{parameter_details.get('cloudformation_stack_output').get('output_key')}"
f"-{account_id_to_use_for_boto3_call}"
f"-{region_to_use_for_boto3_call}"
)
elif parameter_details.get("servicecatalog_provisioned_product_output"):
return (
f"{constants.BOTO3_PARAMETERS}"
f"-servicecatalog_provisioned_product_output"
f"-{parameter_details.get('servicecatalog_provisioned_product_output').get('provisioned_product_name')}"
f"-{parameter_details.get('servicecatalog_provisioned_product_output').get('output_key')}"
f"-{account_id_to_use_for_boto3_call}"
f"-{region_to_use_for_boto3_call}"
)

elif parameter_details.get("boto3"):
return (
f"{constants.BOTO3_PARAMETERS}"
f"-{task.get('section_name')}"
f"-{task.get('item_name')}"
f"-{parameter_name}"
f"-{account_id_to_use_for_boto3_call}"
f"-{region_to_use_for_boto3_call}"
)


def boto3_parameter_handler(
new_tasks, parameter_details, parameter_name, puppet_account_id, task
):
Expand All @@ -18,14 +51,12 @@ def boto3_parameter_handler(
"region", constants.HOME_REGION
).replace("${AWS::Region}", task.get("region"))

boto3_parameter_task_reference = (
f"{constants.BOTO3_PARAMETERS}"
f"-{task.get('section_name')}"
f"-{task.get('item_name')}"
f"-{parameter_name}"
f"-{account_id_to_use_for_boto3_call}"
f"-{region_to_use_for_boto3_call}"
boto3_parameter_task_reference = generate_task_reference(
parameter_details,
account_id_to_use_for_boto3_call,
region_to_use_for_boto3_call,
)

task_execution = task.get("execution", constants.EXECUTION_MODE_DEFAULT)
if task.get(task_execution) in [
constants.EXECUTION_MODE_HUB,
Expand Down Expand Up @@ -59,6 +90,12 @@ def boto3_parameter_handler(
boto3_task["manifest_section_names"].update(task.get("manifest_section_names"))
boto3_task["manifest_item_names"].update(task.get("manifest_item_names"))
boto3_task["manifest_account_ids"].update(task.get("manifest_account_ids"))
boto3_task["dependencies"].extend(task.get("dependencies"))

task["dependencies_by_reference"].append(boto3_parameter_task_reference)

if not task.get("boto3_parameters_tasks_references"):
task["boto3_parameters_tasks_references"] = dict()

task["boto3_parameters_tasks_references"][
parameter_name
] = boto3_parameter_task_reference
15 changes: 8 additions & 7 deletions ...catalog_puppet/commands/task_reference_helpers/generators/boto3_parameter_handler_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ def test_for_boto3_parameters(self):

describe_stacks_filter = "xyz"

output_key = "SomeOutputKey"

parameter_details = {
"boto3": {
"account_id": "${AWS::AccountId}",
Expand All @@ -74,7 +76,7 @@ def test_for_boto3_parameters(self):
},
"cloudformation_stack_output": {
"account_id": "${AWS::AccountId}",
"output_key": "SomeOutputKey",
"output_key": output_key,
"region": "${AWS::Region}",
"stack_name": dependency_item_name,
},
Expand All @@ -89,12 +91,13 @@ def test_for_boto3_parameters(self):

# verify
n_new_tasks = len(new_tasks.keys())
boto3_parameter_task_reference = f"{constants.BOTO3_PARAMETERS}-cloudformation_stack_output-{dependency_item_name}-{output_key}-{account_id}-{region}"
self.assertEqual(
[
"create-policies",
f"prepare-account-for-stacks-{account_id}",
f"get-template-from-s3-stacks-{item_name}",
f"{constants.BOTO3_PARAMETERS}-{section_name}-{item_name}-{parameter_name}-{account_id}-{region}",
boto3_parameter_task_reference,
],
new_tasks[task_reference].get("dependencies_by_reference"),
"assert new dependency is added to the task needing the parameter",
Expand All @@ -105,11 +108,9 @@ def test_for_boto3_parameters(self):
{
"status": None,
"execution": "spoke",
"task_reference": expected_boto3_task_ref,
"task_reference": boto3_parameter_task_reference,
"dependencies_by_reference": [],
"dependencies": [
{"affinity": "stack", "name": dependency_item_name, "type": "stack"}
],
"dependencies": [],
"manifest_section_names": {constants.STACKS: True},
"manifest_item_names": {item_name: True},
"manifest_account_ids": {account_id: True},
Expand All @@ -122,7 +123,7 @@ def test_for_boto3_parameters(self):
"use_paginator": True,
"section_name": constants.BOTO3_PARAMETERS,
},
new_tasks[expected_boto3_task_ref],
new_tasks[boto3_parameter_task_reference],
"assert the boto3 task is generated correctly",
)

Expand Down
73 changes: 42 additions & 31 deletions servicecatalog_puppet/commands/task_reference_helpers/generators/ssm_outputs_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,50 +7,54 @@
def ssm_outputs_handler(
all_tasks,
all_tasks_task_reference,
default_region,
home_region,
item_name,
puppet_account_id,
section_name,
task_to_add,
):
for ssm_parameter_output in task_to_add.get("ssm_param_outputs", []):
output_region = ssm_parameter_output.get("region", default_region)
task_account_id = task_to_add.get("account_id")
output_account_id = ssm_parameter_output.get(
"account_id", puppet_account_id
).replace("${AWS::AccountId}", task_account_id)
ssm_parameter_output_task_reference = f'{constants.SSM_OUTPUTS}-{output_account_id}-{output_region}-{ssm_parameter_output.get("param_name")}'
ssm_parameter_output_task_reference = ssm_parameter_output_task_reference.replace(
"${AWS::Region}", task_to_add.get("region")
).replace(
"${AWS::AccountId}", task_account_id
task_region = task_to_add.get("region")

output_region = ssm_parameter_output.get("region", home_region).replace(
"${AWS::Region}", task_region
)
output_account_id = (
ssm_parameter_output.get("account_id", puppet_account_id)
.replace("${AWS::AccountId}", task_account_id)
.replace("${AWS::PuppetAccountId}", puppet_account_id)
)

output_parameter_name = (
ssm_parameter_output.get("param_name")
.replace("${AWS::Region}", task_region)
.replace("${AWS::AccountId}", task_account_id)
)

ssm_parameter_output_task_reference = f"{constants.SSM_OUTPUTS}-{output_account_id}-{output_region}-{output_parameter_name}"

if all_tasks.get(ssm_parameter_output_task_reference):
raise Exception(
f"You have two tasks outputting the same SSM parameter output: {ssm_parameter_output.get('param_name')}: {ssm_parameter_output_task_reference}"
)

else:
all_tasks[ssm_parameter_output_task_reference] = dict(
manifest_section_names=dict(),
manifest_item_names=dict(),
manifest_account_ids=dict(),
task_reference=ssm_parameter_output_task_reference,
param_name=ssm_parameter_output.get("param_name")
.replace("${AWS::Region}", task_to_add.get("region"))
.replace("${AWS::AccountId}", task_account_id),
stack_output=ssm_parameter_output.get("stack_output"),
force_operation=ssm_parameter_output.get("force_operation", False),
account_id=output_account_id,
region=output_region,
dependencies_by_reference=[all_tasks_task_reference],
task_generating_output=all_tasks_task_reference,
status=task_to_add.get("status"),
section_name=constants.SSM_OUTPUTS,
execution=task_to_add.get(
"execution", constants.EXECUTION_MODE_DEFAULT
),
)
all_tasks[ssm_parameter_output_task_reference] = dict(
manifest_section_names=dict(),
manifest_item_names=dict(),
manifest_account_ids=dict(),
task_reference=ssm_parameter_output_task_reference,
param_name=output_parameter_name,
stack_output=ssm_parameter_output.get("stack_output"),
force_operation=ssm_parameter_output.get("force_operation", False),
account_id=output_account_id,
region=output_region,
dependencies_by_reference=[all_tasks_task_reference],
task_generating_output=all_tasks_task_reference,
status=task_to_add.get("status"),
section_name=constants.SSM_OUTPUTS,
execution=task_to_add.get("execution", constants.EXECUTION_MODE_DEFAULT),
)
all_tasks[ssm_parameter_output_task_reference]["manifest_section_names"][
section_name
] = True
Expand All @@ -60,3 +64,10 @@ def ssm_outputs_handler(
all_tasks[ssm_parameter_output_task_reference]["manifest_account_ids"][
output_account_id
] = True

if not task_to_add.get("ssm_outputs_tasks_references"):
task_to_add["ssm_outputs_tasks_references"] = dict()

task_to_add["ssm_outputs_tasks_references"][
output_parameter_name
] = ssm_parameter_output_task_reference

0 comments on commit 830c39a

Please sign in to comment.