Skip to content

Commit

Permalink
feat: Resolve Intrinsics Functions in template (#1333)
Browse files Browse the repository at this point in the history
  • Loading branch information
@viksrivat @jfuss
viksrivat authored and jfuss committed Aug 14, 2019
1 parent 71e5239 commit c09f470
Show file tree
Hide file tree
Showing 11 changed files with 942 additions and 372 deletions.
1 change: 1 addition & 0 deletions samcli/commands/local/lib/api_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ def _extract_api(self, resources):
---------
An Api from the parsed template
"""

collector = ApiCollector()
provider = self.find_api_provider(resources)
provider.extract_resources(resources, collector, cwd=self.cwd)
Expand Down
1 change: 1 addition & 0 deletions samcli/commands/local/lib/cfn_api_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ def extract_resources(self, resources, collector, cwd=None):
-------
Returns a list of routes
"""

for logical_id, resource in resources.items():
resource_type = resource.get(CfnBaseApiProvider.RESOURCE_TYPE)
if resource_type == CfnApiProvider.APIGATEWAY_RESTAPI:
Expand Down
67 changes: 15 additions & 52 deletions samcli/commands/local/lib/sam_base_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@

import logging

from samtranslator.intrinsics.resolver import IntrinsicsResolver
from samtranslator.intrinsics.actions import RefAction

from samcli.lib.samlib.wrapper import SamTranslatorWrapper
from samcli.lib.intrinsic_resolver.intrinsic_property_resolver import IntrinsicResolver
from samcli.lib.intrinsic_resolver.intrinsics_symbol_table import IntrinsicsSymbolTable
from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer
from samcli.lib.samlib.wrapper import SamTranslatorWrapper

LOG = logging.getLogger(__name__)

Expand All @@ -18,24 +17,6 @@ class SamBaseProvider(object):
Base class for SAM Template providers
"""

# There is not much benefit in infering real values for these parameters in local development context. These values
# are usually representative of an AWS environment and stack, but in local development scenario they don't make
# sense. If customers choose to, they can always override this value through the CLI interface.
DEFAULT_PSEUDO_PARAM_VALUES = {
"AWS::AccountId": "123456789012",
"AWS::Partition": "aws",

"AWS::Region": "us-east-1",

"AWS::StackName": "local",
"AWS::StackId": "arn:aws:cloudformation:us-east-1:123456789012:stack/"
"local/51af3dc0-da77-11e4-872e-1234567db123",
"AWS::URLSuffix": "localhost"
}

# Only Ref is supported when resolving template parameters
_SUPPORTED_INTRINSICS = [RefAction]

@staticmethod
def get_template(template_dict, parameter_overrides=None):
"""
Expand All @@ -59,37 +40,19 @@ def get_template(template_dict, parameter_overrides=None):
template_dict = template_dict or {}
if template_dict:
template_dict = SamTranslatorWrapper(template_dict).run_plugins()

template_dict = SamBaseProvider._resolve_parameters(template_dict, parameter_overrides)
ResourceMetadataNormalizer.normalize(template_dict)
return template_dict

@staticmethod
def _resolve_parameters(template_dict, parameter_overrides):
"""
In the given template, apply parameter values to resolve intrinsic functions
Parameters
----------
template_dict : dict
SAM Template
logical_id_translator = SamBaseProvider._get_parameter_values(
template_dict, parameter_overrides
)
resolver = IntrinsicResolver(
template=template_dict,
symbol_resolver=IntrinsicsSymbolTable(
logical_id_translator=logical_id_translator, template=template_dict
),
)
template_dict = resolver.resolve_template(ignore_errors=True)

parameter_overrides : dict
Values for template parameters provided by user
Returns
-------
dict
Resolved SAM template
"""

parameter_values = SamBaseProvider._get_parameter_values(template_dict, parameter_overrides)

supported_intrinsics = {action.intrinsic_name: action() for action in SamBaseProvider._SUPPORTED_INTRINSICS}

# Intrinsics resolver will mutate the original template
return IntrinsicsResolver(parameters=parameter_values, supported_intrinsics=supported_intrinsics) \
.resolve_parameter_refs(template_dict)
return template_dict

@staticmethod
def _get_parameter_values(template_dict, parameter_overrides):
Expand All @@ -116,7 +79,7 @@ def _get_parameter_values(template_dict, parameter_overrides):
# NOTE: Ordering of following statements is important. It makes sure that any user-supplied values
# override the defaults
parameter_values = {}
parameter_values.update(SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES)
parameter_values.update(IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES)
parameter_values.update(default_values)
parameter_values.update(parameter_overrides or {})

Expand Down
70 changes: 38 additions & 32 deletions samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import base64
import re
from collections import OrderedDict

from six import string_types

Expand Down Expand Up @@ -82,6 +83,7 @@ def __init__(self, template, symbol_resolver):
self._mapping = None
self._parameters = None
self._conditions = None
self._outputs = None
self.init_template(template)

self._symbol_resolver = symbol_resolver
Expand All @@ -95,6 +97,7 @@ def init_template(self, template):
self._mapping = self._template.get("Mappings", {})
self._parameters = self._template.get("Parameters", {})
self._conditions = self._template.get("Conditions", {})
self._outputs = self._template.get("Outputs", {})

def default_intrinsic_function_map(self):
"""
Expand Down Expand Up @@ -217,52 +220,65 @@ def intrinsic_property_resolver(self, intrinsic, parent_function="template"):
# resolve each of it's sub properties.
sanitized_dict = {}
for key, val in intrinsic.items():
sanitized_key = self.intrinsic_property_resolver(
key, parent_function=parent_function
)
sanitized_val = self.intrinsic_property_resolver(
val, parent_function=parent_function
)
verify_intrinsic_type_str(
sanitized_key,
message="The keys of the dictionary {} in {} must all resolve to a string".format(
sanitized_key, parent_function
),
)
sanitized_key = self.intrinsic_property_resolver(key, parent_function=parent_function)
sanitized_val = self.intrinsic_property_resolver(val, parent_function=parent_function)
verify_intrinsic_type_str(sanitized_key,
message="The keys of the dictionary {} in {} must all resolve to a string".format(
sanitized_key, parent_function
))
sanitized_dict[sanitized_key] = sanitized_val
return sanitized_dict

def resolve_template(self, ignore_errors=False):
"""
This will parse through every entry in a CloudFormation template and resolve them based on the symbol_resolver.
This resolves all the attributes of the CloudFormation dictionary Resources, Outputs, Mappings, Parameters,
Conditions.
Return
-------
Return a processed template
"""
processed_template = OrderedDict()
processed_template["Resources"] = self.resolve_attribute(self._resources, ignore_errors)
processed_template["Outputs"] = self.resolve_attribute(self._outputs, ignore_errors)
processed_template["Mappings"] = self.resolve_attribute(self._resources, ignore_errors)
processed_template["Parameters"] = self.resolve_attribute(self._resources, ignore_errors)
processed_template["Conditions"] = self.resolve_attribute(self._resources, ignore_errors)
return processed_template

def resolve_attribute(self, cloud_formation_property, ignore_errors=False):
"""
This will parse through every entry in a CloudFormation root key and resolve them based on the symbol_resolver.
Customers can optionally ignore resource errors and default to whatever the resource provides.
Parameters
-----------
cloud_formation_property: dict
A high Level dictionary containg either the Mappings, Resources, Outputs, or Parameters Dictionary
ignore_errors: bool
An option to ignore errors that are InvalidIntrinsicException and InvalidSymbolException
Return
-------
A resolved template with all references possible simplified
"""
processed_template = {}
for key, val in self._resources.items():
processed_dict = OrderedDict()
for key, val in cloud_formation_property.items():
processed_key = self._symbol_resolver.get_translation(key) or key
try:
processed_resource = self.intrinsic_property_resolver(val)
processed_template[processed_key] = processed_resource
processed_dict[processed_key] = processed_resource
except (InvalidIntrinsicException, InvalidSymbolException) as e:
resource_type = val.get("Type", "")
if ignore_errors:
LOG.error(
"Unable to process properties of %s.%s", key, resource_type
)
processed_template[key] = val
processed_dict[key] = val
else:
raise InvalidIntrinsicException(
"Exception with property of {}.{}".format(key, resource_type) + ": " + str(e.args)
)
return processed_template
return processed_dict

def handle_fn_join(self, intrinsic_value):
"""
Expand All @@ -280,21 +296,15 @@ def handle_fn_join(self, intrinsic_value):
-------
A string with the resolved attributes
"""
arguments = self.intrinsic_property_resolver(
intrinsic_value, parent_function=IntrinsicResolver.FN_JOIN
)
arguments = self.intrinsic_property_resolver(intrinsic_value, parent_function=IntrinsicResolver.FN_JOIN)

verify_intrinsic_type_list(arguments, IntrinsicResolver.FN_JOIN)

delimiter = arguments[0]

verify_intrinsic_type_str(
delimiter, IntrinsicResolver.FN_JOIN, position_in_list="first"
)
verify_intrinsic_type_str(delimiter, IntrinsicResolver.FN_JOIN, position_in_list="first")

value_list = self.intrinsic_property_resolver(
arguments[1], parent_function=IntrinsicResolver.FN_JOIN
)
value_list = self.intrinsic_property_resolver(arguments[1], parent_function=IntrinsicResolver.FN_JOIN)

verify_intrinsic_type_list(
value_list,
Expand Down Expand Up @@ -530,7 +540,7 @@ def handle_fn_get_azs(self, intrinsic_value):
verify_intrinsic_type_str(intrinsic_value, IntrinsicResolver.FN_GET_AZS)

if intrinsic_value == "":
intrinsic_value = self._symbol_resolver.DEFAULT_REGION
intrinsic_value = self._symbol_resolver.handle_pseudo_region()

if intrinsic_value not in self._symbol_resolver.REGIONS:
raise InvalidIntrinsicException(
Expand Down Expand Up @@ -961,7 +971,6 @@ def handle_fn_or(self, intrinsic_value):
intrinsic_value, parent_function=IntrinsicResolver.FN_OR
)
verify_intrinsic_type_list(arguments, IntrinsicResolver.FN_OR)

for i, argument in enumerate(arguments):
if isinstance(argument, dict) and "Condition" in argument:
condition_name = argument.get("Condition")
Expand All @@ -978,16 +987,13 @@ def handle_fn_or(self, intrinsic_value):
condition, parent_function=IntrinsicResolver.FN_OR
)
verify_intrinsic_type_bool(condition_evaluated, IntrinsicResolver.FN_OR)

if condition_evaluated:
return True
else:
condition = self.intrinsic_property_resolver(
argument, parent_function=IntrinsicResolver.FN_OR
)
verify_intrinsic_type_bool(condition, IntrinsicResolver.FN_OR)

if condition:
return True

return False
27 changes: 19 additions & 8 deletions samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

from six import string_types

from samcli.commands.local.lib.sam_base_provider import SamBaseProvider
from samcli.lib.intrinsic_resolver.intrinsic_property_resolver import IntrinsicResolver
from samcli.lib.intrinsic_resolver.invalid_intrinsic_exception import (
InvalidSymbolException,
Expand All @@ -32,7 +31,19 @@ class IntrinsicsSymbolTable(object):
AWS_NOVALUE,
]

DEFAULT_REGION = "us-east-1"
# There is not much benefit in infering real values for these parameters in local development context. These values
# are usually representative of an AWS environment and stack, but in local development scenario they don't make
# sense. If customers choose to, they can always override this value through the CLI interface.
DEFAULT_PSEUDO_PARAM_VALUES = {
"AWS::AccountId": "123456789012",
"AWS::Partition": "aws",
"AWS::Region": "us-east-1",
"AWS::StackName": "local",
"AWS::StackId": "arn:aws:cloudformation:us-east-1:123456789012:stack/"
"local/51af3dc0-da77-11e4-872e-1234567db123",
"AWS::URLSuffix": "localhost",
}

REGIONS = {
"us-east-1": [
"us-east-1a",
Expand Down Expand Up @@ -207,11 +218,11 @@ def resolve_symbols(self, logical_id, resource_attribute, ignore_errors=False):
translated = self._parameters.get(logical_id, {}).get("Default")
if translated:
return translated

# Handle Default Property Type Resolution
resource_type = self._resources.get(logical_id, {}).get(
IntrinsicsSymbolTable.CFN_RESOURCE_TYPE
)

resolver = (
self.default_type_resolver.get(resource_type, {}).get(resource_attribute)
if resource_type
Expand Down Expand Up @@ -290,7 +301,7 @@ def get_translation(self, logical_id, resource_attributes=IntrinsicResolver.REF)
"""
logical_id_item = self.logical_id_translator.get(logical_id, {})
if isinstance(logical_id_item, string_types):
if any(isinstance(logical_id_item, object_type) for object_type in [string_types, list, bool, int]):
if (
resource_attributes != IntrinsicResolver.REF and resource_attributes != ""
):
Expand Down Expand Up @@ -322,7 +333,7 @@ def handle_pseudo_account_id():
-------
A pseudo account id
"""
return SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES.get(
return IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.AWS_ACCOUNT_ID
)

Expand All @@ -338,7 +349,7 @@ def handle_pseudo_region(self):
"""
return (
self.logical_id_translator.get(IntrinsicsSymbolTable.AWS_REGION) or os.getenv("AWS_REGION") or
SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.AWS_REGION
)
)
Expand Down Expand Up @@ -385,7 +396,7 @@ def handle_pseudo_stack_id():
-------
A randomized string
"""
return SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES.get(
return IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.AWS_STACK_ID
)

Expand All @@ -400,7 +411,7 @@ def handle_pseudo_stack_name():
-------
A randomized string
"""
return SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES.get(
return IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.AWS_STACK_NAME
)

Expand Down
Loading

0 comments on commit c09f470

Please sign in to comment.