Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Integrate Intrinsics #1333

Merged
merged 12 commits into from
Aug 14, 2019
Merged

Integrate Intrinsics #1333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
de55381
Add 2 Intrinsics Integration Tests
viksrivat Aug 12, 2019
a78f309
Fix cyclic imports
viksrivat Aug 12, 2019
499260c
Merge branch 'develop' into feature/integrate_intrinsics
viksrivat Aug 12, 2019
bc70000
Update intrinsics integration tests
viksrivat Aug 12, 2019
3c0d13e
Change processed_template to OrderedDict
viksrivat Aug 12, 2019
0e6b367
Update resolve_tempalte to actually resolve the template
viksrivat Aug 12, 2019
60f37ae
Add Logical ID Translator to SymbolTable
viksrivat Aug 13, 2019
9aae94d
Remove resolve_paremeters section
viksrivat Aug 13, 2019
37093d2
Update Tests and translation
viksrivat Aug 13, 2019
b965402
Remove resolve_parameters
viksrivat Aug 13, 2019
c952b7e
Remove unneeded intrinsics import
viksrivat Aug 14, 2019
486daf4
Merge branch 'develop' into feature/integrate_intrinsics
jfuss Aug 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions samcli/commands/local/lib/api_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ def _extract_api(self, resources):
---------
An Api from the parsed template
"""

collector = ApiCollector()
provider = self.find_api_provider(resources)
provider.extract_resources(resources, collector, cwd=self.cwd)
Expand Down
1 change: 1 addition & 0 deletions samcli/commands/local/lib/cfn_api_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ def extract_resources(self, resources, collector, cwd=None):
-------
Returns a list of routes
"""

for logical_id, resource in resources.items():
resource_type = resource.get(CfnBaseApiProvider.RESOURCE_TYPE)
if resource_type == CfnApiProvider.APIGATEWAY_RESTAPI:
Expand Down
67 changes: 15 additions & 52 deletions samcli/commands/local/lib/sam_base_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,10 @@

import logging

from samtranslator.intrinsics.resolver import IntrinsicsResolver
from samtranslator.intrinsics.actions import RefAction

from samcli.lib.samlib.wrapper import SamTranslatorWrapper
from samcli.lib.intrinsic_resolver.intrinsic_property_resolver import IntrinsicResolver
from samcli.lib.intrinsic_resolver.intrinsics_symbol_table import IntrinsicsSymbolTable
from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer
from samcli.lib.samlib.wrapper import SamTranslatorWrapper

LOG = logging.getLogger(__name__)

Expand All @@ -18,24 +17,6 @@ class SamBaseProvider(object):
Base class for SAM Template providers
"""

# There is not much benefit in infering real values for these parameters in local development context. These values
# are usually representative of an AWS environment and stack, but in local development scenario they don't make
# sense. If customers choose to, they can always override this value through the CLI interface.
DEFAULT_PSEUDO_PARAM_VALUES = {
"AWS::AccountId": "123456789012",
"AWS::Partition": "aws",

"AWS::Region": "us-east-1",

"AWS::StackName": "local",
"AWS::StackId": "arn:aws:cloudformation:us-east-1:123456789012:stack/"
"local/51af3dc0-da77-11e4-872e-1234567db123",
"AWS::URLSuffix": "localhost"
}

# Only Ref is supported when resolving template parameters
_SUPPORTED_INTRINSICS = [RefAction]

@staticmethod
def get_template(template_dict, parameter_overrides=None):
"""
Expand All @@ -59,37 +40,19 @@ def get_template(template_dict, parameter_overrides=None):
template_dict = template_dict or {}
if template_dict:
template_dict = SamTranslatorWrapper(template_dict).run_plugins()

template_dict = SamBaseProvider._resolve_parameters(template_dict, parameter_overrides)
ResourceMetadataNormalizer.normalize(template_dict)
return template_dict

@staticmethod
def _resolve_parameters(template_dict, parameter_overrides):
"""
In the given template, apply parameter values to resolve intrinsic functions

Parameters
----------
template_dict : dict
SAM Template
logical_id_translator = SamBaseProvider._get_parameter_values(
template_dict, parameter_overrides
)
resolver = IntrinsicResolver(
template=template_dict,
symbol_resolver=IntrinsicsSymbolTable(
logical_id_translator=logical_id_translator, template=template_dict
),
)
template_dict = resolver.resolve_template(ignore_errors=True)

parameter_overrides : dict
Values for template parameters provided by user

Returns
-------
dict
Resolved SAM template
"""

parameter_values = SamBaseProvider._get_parameter_values(template_dict, parameter_overrides)

supported_intrinsics = {action.intrinsic_name: action() for action in SamBaseProvider._SUPPORTED_INTRINSICS}

# Intrinsics resolver will mutate the original template
return IntrinsicsResolver(parameters=parameter_values, supported_intrinsics=supported_intrinsics) \
.resolve_parameter_refs(template_dict)
return template_dict

@staticmethod
def _get_parameter_values(template_dict, parameter_overrides):
Expand All @@ -116,7 +79,7 @@ def _get_parameter_values(template_dict, parameter_overrides):
# NOTE: Ordering of following statements is important. It makes sure that any user-supplied values
# override the defaults
parameter_values = {}
parameter_values.update(SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES)
parameter_values.update(IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES)
parameter_values.update(default_values)
parameter_values.update(parameter_overrides or {})

Expand Down
70 changes: 38 additions & 32 deletions samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import base64
import re
from collections import OrderedDict

from six import string_types

Expand Down Expand Up @@ -82,6 +83,7 @@ def __init__(self, template, symbol_resolver):
self._mapping = None
self._parameters = None
self._conditions = None
self._outputs = None
self.init_template(template)

self._symbol_resolver = symbol_resolver
Expand All @@ -95,6 +97,7 @@ def init_template(self, template):
self._mapping = self._template.get("Mappings", {})
self._parameters = self._template.get("Parameters", {})
self._conditions = self._template.get("Conditions", {})
self._outputs = self._template.get("Outputs", {})

def default_intrinsic_function_map(self):
"""
Expand Down Expand Up @@ -217,52 +220,65 @@ def intrinsic_property_resolver(self, intrinsic, parent_function="template"):
# resolve each of it's sub properties.
sanitized_dict = {}
for key, val in intrinsic.items():
sanitized_key = self.intrinsic_property_resolver(
key, parent_function=parent_function
)
sanitized_val = self.intrinsic_property_resolver(
val, parent_function=parent_function
)
verify_intrinsic_type_str(
sanitized_key,
message="The keys of the dictionary {} in {} must all resolve to a string".format(
sanitized_key, parent_function
),
)
sanitized_key = self.intrinsic_property_resolver(key, parent_function=parent_function)
sanitized_val = self.intrinsic_property_resolver(val, parent_function=parent_function)
verify_intrinsic_type_str(sanitized_key,
message="The keys of the dictionary {} in {} must all resolve to a string".format(
sanitized_key, parent_function
))
sanitized_dict[sanitized_key] = sanitized_val
return sanitized_dict

def resolve_template(self, ignore_errors=False):
"""
This will parse through every entry in a CloudFormation template and resolve them based on the symbol_resolver.
This resolves all the attributes of the CloudFormation dictionary Resources, Outputs, Mappings, Parameters,
Conditions.

Return
-------
Return a processed template
"""
processed_template = OrderedDict()
processed_template["Resources"] = self.resolve_attribute(self._resources, ignore_errors)
processed_template["Outputs"] = self.resolve_attribute(self._outputs, ignore_errors)
processed_template["Mappings"] = self.resolve_attribute(self._resources, ignore_errors)
processed_template["Parameters"] = self.resolve_attribute(self._resources, ignore_errors)
processed_template["Conditions"] = self.resolve_attribute(self._resources, ignore_errors)
return processed_template

def resolve_attribute(self, cloud_formation_property, ignore_errors=False):
"""
This will parse through every entry in a CloudFormation root key and resolve them based on the symbol_resolver.
Customers can optionally ignore resource errors and default to whatever the resource provides.

Parameters
-----------
cloud_formation_property: dict
A high Level dictionary containg either the Mappings, Resources, Outputs, or Parameters Dictionary
ignore_errors: bool
An option to ignore errors that are InvalidIntrinsicException and InvalidSymbolException
Return
-------
A resolved template with all references possible simplified
"""
processed_template = {}
for key, val in self._resources.items():
processed_dict = OrderedDict()
for key, val in cloud_formation_property.items():
processed_key = self._symbol_resolver.get_translation(key) or key
try:
processed_resource = self.intrinsic_property_resolver(val)
processed_template[processed_key] = processed_resource
processed_dict[processed_key] = processed_resource
except (InvalidIntrinsicException, InvalidSymbolException) as e:
resource_type = val.get("Type", "")
if ignore_errors:
LOG.error(
"Unable to process properties of %s.%s", key, resource_type
)
processed_template[key] = val
processed_dict[key] = val
else:
raise InvalidIntrinsicException(
"Exception with property of {}.{}".format(key, resource_type) + ": " + str(e.args)
)
return processed_template
return processed_dict

def handle_fn_join(self, intrinsic_value):
"""
Expand All @@ -280,21 +296,15 @@ def handle_fn_join(self, intrinsic_value):
-------
A string with the resolved attributes
"""
arguments = self.intrinsic_property_resolver(
intrinsic_value, parent_function=IntrinsicResolver.FN_JOIN
)
arguments = self.intrinsic_property_resolver(intrinsic_value, parent_function=IntrinsicResolver.FN_JOIN)

verify_intrinsic_type_list(arguments, IntrinsicResolver.FN_JOIN)

delimiter = arguments[0]

verify_intrinsic_type_str(
delimiter, IntrinsicResolver.FN_JOIN, position_in_list="first"
)
verify_intrinsic_type_str(delimiter, IntrinsicResolver.FN_JOIN, position_in_list="first")

value_list = self.intrinsic_property_resolver(
arguments[1], parent_function=IntrinsicResolver.FN_JOIN
)
value_list = self.intrinsic_property_resolver(arguments[1], parent_function=IntrinsicResolver.FN_JOIN)

verify_intrinsic_type_list(
value_list,
Expand Down Expand Up @@ -530,7 +540,7 @@ def handle_fn_get_azs(self, intrinsic_value):
verify_intrinsic_type_str(intrinsic_value, IntrinsicResolver.FN_GET_AZS)

if intrinsic_value == "":
intrinsic_value = self._symbol_resolver.DEFAULT_REGION
intrinsic_value = self._symbol_resolver.handle_pseudo_region()

if intrinsic_value not in self._symbol_resolver.REGIONS:
raise InvalidIntrinsicException(
Expand Down Expand Up @@ -961,7 +971,6 @@ def handle_fn_or(self, intrinsic_value):
intrinsic_value, parent_function=IntrinsicResolver.FN_OR
)
verify_intrinsic_type_list(arguments, IntrinsicResolver.FN_OR)

for i, argument in enumerate(arguments):
if isinstance(argument, dict) and "Condition" in argument:
condition_name = argument.get("Condition")
Expand All @@ -978,16 +987,13 @@ def handle_fn_or(self, intrinsic_value):
condition, parent_function=IntrinsicResolver.FN_OR
)
verify_intrinsic_type_bool(condition_evaluated, IntrinsicResolver.FN_OR)

if condition_evaluated:
return True
else:
condition = self.intrinsic_property_resolver(
argument, parent_function=IntrinsicResolver.FN_OR
)
verify_intrinsic_type_bool(condition, IntrinsicResolver.FN_OR)

if condition:
return True

return False
27 changes: 19 additions & 8 deletions samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

from six import string_types

from samcli.commands.local.lib.sam_base_provider import SamBaseProvider
from samcli.lib.intrinsic_resolver.intrinsic_property_resolver import IntrinsicResolver
from samcli.lib.intrinsic_resolver.invalid_intrinsic_exception import (
InvalidSymbolException,
Expand All @@ -32,7 +31,19 @@ class IntrinsicsSymbolTable(object):
AWS_NOVALUE,
]

DEFAULT_REGION = "us-east-1"
# There is not much benefit in infering real values for these parameters in local development context. These values
# are usually representative of an AWS environment and stack, but in local development scenario they don't make
# sense. If customers choose to, they can always override this value through the CLI interface.
DEFAULT_PSEUDO_PARAM_VALUES = {
"AWS::AccountId": "123456789012",
"AWS::Partition": "aws",
"AWS::Region": "us-east-1",
"AWS::StackName": "local",
"AWS::StackId": "arn:aws:cloudformation:us-east-1:123456789012:stack/"
"local/51af3dc0-da77-11e4-872e-1234567db123",
"AWS::URLSuffix": "localhost",
}

REGIONS = {
"us-east-1": [
"us-east-1a",
Expand Down Expand Up @@ -207,11 +218,11 @@ def resolve_symbols(self, logical_id, resource_attribute, ignore_errors=False):
translated = self._parameters.get(logical_id, {}).get("Default")
if translated:
return translated

# Handle Default Property Type Resolution
resource_type = self._resources.get(logical_id, {}).get(
IntrinsicsSymbolTable.CFN_RESOURCE_TYPE
)

resolver = (
self.default_type_resolver.get(resource_type, {}).get(resource_attribute)
if resource_type
Expand Down Expand Up @@ -290,7 +301,7 @@ def get_translation(self, logical_id, resource_attributes=IntrinsicResolver.REF)

"""
logical_id_item = self.logical_id_translator.get(logical_id, {})
if isinstance(logical_id_item, string_types):
if any(isinstance(logical_id_item, object_type) for object_type in [string_types, list, bool, int]):
if (
resource_attributes != IntrinsicResolver.REF and resource_attributes != ""
):
Expand Down Expand Up @@ -322,7 +333,7 @@ def handle_pseudo_account_id():
-------
A pseudo account id
"""
return SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES.get(
return IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.AWS_ACCOUNT_ID
)

Expand All @@ -338,7 +349,7 @@ def handle_pseudo_region(self):
"""
return (
self.logical_id_translator.get(IntrinsicsSymbolTable.AWS_REGION) or os.getenv("AWS_REGION") or
SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.AWS_REGION
)
)
Expand Down Expand Up @@ -385,7 +396,7 @@ def handle_pseudo_stack_id():
-------
A randomized string
"""
return SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES.get(
return IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.AWS_STACK_ID
)

Expand All @@ -400,7 +411,7 @@ def handle_pseudo_stack_name():
-------
A randomized string
"""
return SamBaseProvider.DEFAULT_PSEUDO_PARAM_VALUES.get(
return IntrinsicsSymbolTable.DEFAULT_PSEUDO_PARAM_VALUES.get(
IntrinsicsSymbolTable.AWS_STACK_NAME
)

Expand Down
Loading