[Resolve #1252] Supporting resolvers in Hook and Resolver arguments, with new !substitute, !join, !split, and !select resolvers!

sceptre/config/reader.py

@@ -182,7 +182,7 @@ def constructor_factory(node_class):
             :rtype: func
             """
 
-            # This function signture is required by PyYAML
+            # This function signature is required by PyYAML
             def class_constructor(loader, node):
                 return node_class(
                     loader.construct_object(self.resolve_node_tag(loader, node))

sceptre/helpers.py

@@ -2,7 +2,7 @@
 from contextlib import contextmanager
 from datetime import datetime
 from os import sep
-from typing import Optional, Any, List
+from typing import Optional, Any, List, Tuple, Union
 
 import dateutil.parser
 import deprecation
@@ -68,6 +68,28 @@ def func_on_instance(key):
     return attr
 
 
+Container = Union[list, dict]
+Key = Union[str, int]
+
+
+def delete_keys_from_containers(keys_to_delete: List[Tuple[Container, Key]]):
+    """Removes the indicated keys/indexes from their paired containers."""
+    list_items_to_delete = []
+    for container, key in keys_to_delete:
+        if isinstance(container, list):
+            # If it's a list, we want to gather up the items to remove from the list.
+            # We don't want to modify the list length yet, since removals will change all the other
+            # list indexes. Instead, we'll get the actual items at those indexes to remove later.
+            list_items_to_delete.append((container, container[key]))
+        else:
+            del container[key]
+
+    # Finally, now that we have all the items we want to remove the lists, we'll remove those
+    # items specifically from the lists.
+    for containing_list, item in list_items_to_delete:
+        containing_list.remove(item)
+
+
 def normalise_path(path):
     """
     Converts a path to use correct path separator.

sceptre/hooks/__init__.py

@@ -7,11 +7,13 @@
 
 from typing import TYPE_CHECKING, Any
 
+from sceptre.resolvers import ResolvableArgumentBase
+
 if TYPE_CHECKING:
     from sceptre.stack import Stack
 
 
-class Hook(abc.ABC):
+class Hook(ResolvableArgumentBase, metaclass=abc.ABCMeta):
     """
     Hook is an abstract base class that should be inherited by all hooks.
 
@@ -20,21 +22,12 @@ class Hook(abc.ABC):
     """
 
     def __init__(self, argument: Any = None, stack: "Stack" = None):
+        super().__init__(argument, stack)
         self.logger = logging.getLogger(__name__)
 
         if stack is not None:
             self.logger = StackLoggerAdapter(self.logger, stack.name)
 
-        self.argument = argument
-        self.stack = stack
-
-    def setup(self):
-        """
-        setup is a method that may be overwritten by inheriting classes. Allows
-        hooks to run so initalisation steps when config is first read.
-        """
-        pass  # pragma: no cover
-
     @abc.abstractmethod
     def run(self):
         """
@@ -43,15 +36,6 @@ def run(self):
         """
         pass  # pragma: no cover
 
-    def clone(self, stack: "Stack") -> "Hook":
-        """
-        Produces a "fresh" copy of the Hook, with the specified stack.
-
-        :param stack: The stack to set on the cloned resolver
-        """
-        clone = type(self)(self.argument, stack)
-        return clone
-
 
 class HookProperty(object):
     """
@@ -84,7 +68,7 @@ def __set__(self, instance: "Stack", value):
         """
 
         def setup(attr, key, value: Hook):
-            attr[key] = clone = value.clone(instance)
+            attr[key] = clone = value.clone_for_stack(instance)
             clone.setup()
 
         _call_func_on_values(setup, value, Hook)

sceptre/resolvers/__init__.py

@@ -5,7 +5,7 @@
 from threading import RLock
 from typing import Any, TYPE_CHECKING, Type, Union, TypeVar
 
-from sceptre.helpers import _call_func_on_values
+from sceptre.helpers import _call_func_on_values, delete_keys_from_containers
 from sceptre.logging import StackLoggerAdapter
 from sceptre.resolvers.placeholders import (
     create_placeholder_value,
@@ -17,13 +17,113 @@
     from sceptre import stack
 
 T_Container = TypeVar("T_Container", bound=Union[dict, list])
+Self = TypeVar("Self")
 
 
 class RecursiveResolve(Exception):
     pass
 
 
-class Resolver(abc.ABC):
+class ResolvableArgumentBase:
+    def __init__(self, argument: Any = None, stack: "stack.Stack" = None):
+        self.stack = stack
+
+        self._argument = argument
+        self._argument_is_resolved = False
+
+    @property
+    def argument(self) -> Any:
+        """This is the resolver's argument.
+
+        This property will resolve all nested resolvers inside the argument, but only if this resolver
+        has a Stack.
+
+        Resolving nested resolvers will result in their values being replaced in the dict/list they
+        were in with their resolved value, so we won't have to resolve them again.
+
+        If this property is accessed BEFORE the resolver has a stack, it will return
+        the raw argument value. This is to safeguard any __init__() behaviors from triggering
+        resolution prematurely.
+        """
+        if self.stack is not None and not self._argument_is_resolved:
+            self._resolve_argument()
+
+        return self._argument
+
+    @argument.setter
+    def argument(self, value):
+        self._argument = value
+
+    def _resolve_argument(self):
+        """Resolves all argument resolvers recursively."""
+
+        keys_to_delete = []
+
+        def resolve(attr, key, obj: Resolver):
+            result = obj.resolve()
+            if result is None:
+                keys_to_delete.append((attr, key))
+            else:
+                attr[key] = result
+
+        _call_func_on_values(resolve, self._argument, Resolver)
+        delete_keys_from_containers(keys_to_delete)
+
+        self._argument_is_resolved = True
+
+    def _setup_nested_resolvers(self):
+        """Ensures all nested resolvers in this resolver's argument are also setup when this
+        resolver's setup method is called.
+        """
+
+        def setup_nested(attr, key, obj: Resolver):
+            obj.setup()
+
+        _call_func_on_values(setup_nested, self._argument, Resolver)
+
+    def _clone(self: Self, stack: "stack.Stack") -> Self:
+        """Recursively clones the resolver and its arguments.
+
+        The returned resolver will have an identical argument that is a different memory reference,
+        so that resolvers inherited from a stack group and applied across multiple stacks are
+        independent of each other.
+
+        Furthermore, all nested resolvers in this resolver's argument will also be cloned to ensure
+        they themselves are also independent and fully configured for the current stack.
+        """
+
+        def recursively_clone(obj):
+            if isinstance(obj, Resolver):
+                return obj._clone(stack)
+            if isinstance(obj, list):
+                return [recursively_clone(item) for item in obj]
+            elif isinstance(obj, dict):
+                return {key: recursively_clone(val) for key, val in obj.items()}
+            return obj
+
+        argument = recursively_clone(self._argument)
+        clone = type(self)(argument, stack)
+        return clone
+
+    def clone_for_stack(self: Self, stack: "stack.Stack") -> Self:
+        """Obtains a clone of the current object, setup and ready for use for a given Stack
+        instance.
+        """
+        clone = self._clone(stack)
+        clone._setup_nested_resolvers()
+        clone.setup()
+        return clone
+
+    def setup(self):
+        """
+        This method is called at during stack initialisation.
+        Implementation of this method in subclasses can be used to do any
+        initial setup of the object.
+        """
+        pass  # pragma: no cover
+
+
+class Resolver(ResolvableArgumentBase, metaclass=abc.ABCMeta):
     """
     Resolver is an abstract base class that should be inherited by all
     Resolvers.
@@ -33,21 +133,12 @@ class Resolver(abc.ABC):
     """
 
     def __init__(self, argument: Any = None, stack: "stack.Stack" = None):
+        super().__init__(argument, stack)
+
         self.logger = logging.getLogger(__name__)
         if stack is not None:
             self.logger = StackLoggerAdapter(self.logger, stack.name)
 
-        self.argument = argument
-        self.stack = stack
-
-    def setup(self):
-        """
-        This method is called at during stack initialisation.
-        Implementation of this method in subclasses can be used to do any
-        initial setup of the object.
-        """
-        pass  # pragma: no cover
-
     @abc.abstractmethod
     def resolve(self):
         """
@@ -58,14 +149,6 @@ def resolve(self):
         """
         pass  # pragma: no cover
 
-    def clone(self, stack: "stack.Stack") -> "Resolver":
-        """
-        Produces a "fresh" copy of the Resolver, with the specified stack.
-
-        :param stack: The stack to set on the cloned resolver
-        """
-        return type(self)(self.argument, stack)
-
 
 class ResolvableProperty(abc.ABC):
     """
@@ -127,23 +210,6 @@ def _no_recursive_get(self, stack: "stack.Stack"):
         finally:
             setattr(stack, get_status_name, False)
 
-    def get_setup_resolver_for_stack(
-        self, stack: "stack.Stack", resolver: Resolver
-    ) -> Resolver:
-        """Obtains a clone of the resolver with the stack set on it and the setup method having
-        been called on it.
-
-        :param stack: The stack to set on the Resolver
-        :param resolver: The Resolver to clone and set up
-        :return: The cloned resolver.
-        """
-        # We clone the resolver when we assign the value so that every stack gets its own resolver
-        # rather than potentially having one resolver instance shared in memory across multiple
-        # stacks.
-        clone = resolver.clone(stack)
-        clone.setup()
-        return clone
-
     @abc.abstractmethod
     def get_resolved_value(
         self, stack: "stack.Stack", stack_class: Type["stack.Stack"]
@@ -254,20 +320,7 @@ def resolve(attr: Union[dict, list], key: Union[int, str], value: Resolver):
 
         container = getattr(stack, self.name)
         _call_func_on_values(resolve, container, Resolver)
-        # Remove keys and indexes from their containers that had resolvers resolve to None.
-        list_items_to_delete = []
-        for attr, key in keys_to_delete:
-            if isinstance(attr, list):
-                # If it's a list, we want to gather up the items to remove from the list.
-                # We don't want to modify the list length yet.
-                # Since removals will change all the other list indexes,
-                # we don't wan't to modify lists yet.
-                list_items_to_delete.append((attr, attr[key]))
-            else:
-                del attr[key]
-
-        for containing_list, item in list_items_to_delete:
-            containing_list.remove(item)
+        delete_keys_from_containers(keys_to_delete)
 
         return container
 
@@ -294,7 +347,7 @@ def _clone_container_with_resolvers(
 
         def recurse(obj):
             if isinstance(obj, Resolver):
-                return self.get_setup_resolver_for_stack(stack, obj)
+                return obj.clone_for_stack(stack)
             if isinstance(obj, list):
                 return [recurse(item) for item in obj]
             elif isinstance(obj, dict):
@@ -389,5 +442,5 @@ def assign_value_to_stack(self, stack: "stack.Stack", value: Any):
         :param value: The value to set
         """
         if isinstance(value, Resolver):
-            value = self.get_setup_resolver_for_stack(stack, value)
+            value = value.clone_for_stack(stack)
         setattr(stack, self.name, value)

sceptre/resolvers/join.py

@@ -0,0 +1,8 @@
+from sceptre.resolvers import Resolver
+
+
+class Join(Resolver):
+    def resolve(self):
+        delimiter, items_list = self.argument
+        joined = delimiter.join(items_list)
+        return joined

sceptre/resolvers/select.py

@@ -0,0 +1,7 @@
+from sceptre.resolvers import Resolver
+
+
+class Select(Resolver):
+    def resolve(self):
+        index, items = self.argument
+        return items[index]

sceptre/resolvers/split.py

@@ -0,0 +1,7 @@
+from sceptre.resolvers import Resolver
+
+
+class Split(Resolver):
+    def resolve(self):
+        split_on, split_string = self.argument
+        return split_string.split(split_on)

sceptre/resolvers/substitute.py

@@ -0,0 +1,7 @@
+from sceptre.resolvers import Resolver
+
+
+class Substitute(Resolver):
+    def resolve(self):
+        template, variables = self.argument
+        return template.format(**variables)

setup.py

@@ -71,7 +71,11 @@ def get_version(rel_path):
             "stack_output_external ="
             "sceptre.resolvers.stack_output:StackOutputExternal",
             "no_value = sceptre.resolvers.no_value:NoValue",
+            "select = scpetre.resolvers.select:Select",
             "stack_attr = sceptre.resolvers.stack_attr:StackAttr",
+            "substitute = sceptre.resolvers.substitute:Substitute",
+            "split = sceptre.resolvers.split:Split",
+            "join = sceptre.resolvers.join:Join",
         ],
         "sceptre.template_handlers": [
             "file = sceptre.template_handlers.file:File",