[Resolves #813] Fix recursive config render (#1083)

This PR fixes nested `config.yaml` jinja substitutions (e.g `{{ someKey }}`).

Fixes issue: #813

#### Scenario

Currently, when the project structure has multiple nested config files and one of them tries to use a property from another config file sometimes works, and sometimes not, when not work the error message is `<prop> is undefined.`.

Example structure:

```text
root/
  config/
    prod/
      A.yaml
      config.yaml
      2/
        B.yaml
        config.yaml
        3/
          C.yaml
          config.yaml
```

`config/prod/config.yaml`
```yaml
keyA: A
```

`config/prod/2/config.yaml`
```yaml
keyB: {{ keyA }}-B
```

`config/prod/2/3/config.yaml`
```yaml
keyA: {{ keyB }}-C
```

`keyB` should render the value `A-B` and `keyC` should render the value `A-B-C`

When running `sceptre launch prod` command the `ConfigReader` class doesn't follow a correct order to process the files, and sometimes the config object already has all the properties, sometimes not.

#### Solution

Sceptre already has a concept of recursive read in the `ConfigReader` class that reads all the `config.yaml` files recursively based on the parent path, and already has the config `dict` object with all the properties in the correct place, but when call the `_render` function just pass the `stack_group_config` that only contains the processed `stack_group_config`, which is fine when the files follow the correct order to be processed.

However, if the `ConfigReader` combines the current recursive object with the `stack_group_config` the `_render` will have all the required properties, doesn't matter which file order will be used to process the stacks.

integration-tests/features/create-stack.feature

@@ -48,3 +48,14 @@ Feature: Create stack
     and the template for stack "10/A" is "sam_template.yaml"
     When the user creates stack "10/A"
     Then stack "10/A" exists in "CREATE_COMPLETE" state
+
+  Scenario: create new stack with nested config jinja resolver
+    Given stack_group "12/1" does not exist
+    When the user launches stack_group "12/1"
+    Then all the stacks in stack_group "12/1" are in "CREATE_COMPLETE"
+    and stack "12/1/A" has "Project" tag with "A" value
+    and stack "12/1/A" has "Key" tag with "A" value
+    and stack "12/1/2/B" has "Project" tag with "B" value
+    and stack "12/1/2/B" has "Key" tag with "A-B" value
+    and stack "12/1/2/3/C" has "Project" tag with "C" value
+    and stack "12/1/2/3/C" has "Key" tag with "A-B-C" value

integration-tests/sceptre-project/config/12/1/2/3/C.yaml

@@ -0,0 +1,4 @@
+template_path: valid_template.json
+stack_tags:
+  Project: '{{ project }}'
+  Key: '{{ keyC }}'

integration-tests/sceptre-project/config/12/1/2/3/config.yaml

@@ -0,0 +1,2 @@
+keyC: "{{ keyB }}-C"
+project: C

integration-tests/sceptre-project/config/12/1/2/B.yaml

@@ -0,0 +1,4 @@
+template_path: valid_template.json
+stack_tags:
+  Project: '{{ project }}'
+  Key: '{{ keyB }}'

integration-tests/sceptre-project/config/12/1/2/config.yaml

@@ -0,0 +1,2 @@
+keyB: '{{ keyA }}-B'
+project: B

integration-tests/sceptre-project/config/12/1/A.yaml

@@ -0,0 +1,4 @@
+template_path: valid_template.json
+stack_tags:
+  Key: '{{ keyA }}'
+  Project: '{{ project }}'

integration-tests/sceptre-project/config/12/1/config.yaml

@@ -0,0 +1,2 @@
+keyA: A
+project: A

integration-tests/steps/stacks.py

@@ -311,6 +311,16 @@ def step_impl(context, stack_name, desired_status):
     assert (status == desired_status)
 
 
+@then('stack "{stack_name}" has "{tag_name}" tag with "{desired_tag_value}" value')
+def step_impl(context, stack_name, tag_name, desired_tag_value):
+    full_name = get_cloudformation_stack_name(context, stack_name)
+
+    tags = get_stack_tags(context, full_name)
+    tag = next((tag for tag in tags if tag['Key'] == tag_name), {'Value': None})
+
+    assert (tag['Value'] == desired_tag_value)
+
+
 @then('stack "{stack_name}" does not exist')
 def step_impl(context, stack_name):
     full_name = get_cloudformation_stack_name(context, stack_name)
@@ -352,6 +362,24 @@ def step_impl(context, stack_name, dependant_stack_name, desired_state):
     assert (dep_status == desired_state)
 
 
+def get_stack_tags(context, stack_name, region_name=None):
+    if region_name is not None:
+        stack = boto3.resource('cloudformation', region_name=region_name).Stack
+    else:
+        stack = context.cloudformation.Stack
+
+    try:
+        stack = retry_boto_call(stack, stack_name)
+        retry_boto_call(stack.load)
+        return stack.tags
+    except ClientError as e:
+        if e.response['Error']['Code'] == 'ValidationError' \
+                and e.response['Error']['Message'].endswith("does not exist"):
+            return None
+        else:
+            raise e
+
+
 def get_stack_status(context, stack_name, region_name=None):
     if region_name is not None:
         Stack = boto3.resource('cloudformation', region_name=region_name).Stack

sceptre/config/reader.py

@@ -361,8 +361,12 @@ def _recursive_read(self, directory_path, filename, stack_group_config):
         if directory_path:
             config = self._recursive_read(parent_directory, filename, stack_group_config)
 
+        # Combine the stack_group_config with the nested config dict
+        config_group = stack_group_config.copy()
+        config_group.update(config)
+
         # Read config file and overwrite inherited properties
-        child_config = self._render(directory_path, filename, stack_group_config) or {}
+        child_config = self._render(directory_path, filename, config_group) or {}
 
         for config_key, strategy in CONFIG_MERGE_STRATEGIES.items():
             value = strategy(

tests/test_config_reader.py

@@ -94,6 +94,75 @@ def test_read_reads_config_file(self, filepaths, target):
             "filepath": target
         }
 
+    def test_read_nested_configs(self):
+        with self.runner.isolated_filesystem():
+            project_path = os.path.abspath('./example')
+            config_dir = os.path.join(project_path, "config")
+            stack_group_dir_a = os.path.join(config_dir, "A")
+            stack_group_dir_b = os.path.join(stack_group_dir_a, "B")
+            stack_group_dir_c = os.path.join(stack_group_dir_b, "C")
+
+            os.makedirs(stack_group_dir_c)
+            config_filename = "config.yaml"
+
+            config_a = {"keyA": "A", "shared": "A"}
+            with open(os.path.join(stack_group_dir_a, config_filename), 'w') as\
+                    config_file:
+                yaml.safe_dump(
+                    config_a, stream=config_file, default_flow_style=False
+                )
+
+            config_b = {"keyB": "B", "parent": "{{ keyA }}", "shared": "B"}
+            with open(os.path.join(stack_group_dir_b, config_filename), 'w') as\
+                    config_file:
+                yaml.safe_dump(
+                    config_b, stream=config_file, default_flow_style=False
+                )
+
+            config_c = {"keyC": "C", "parent": "{{ keyB }}", "shared": "C"}
+            with open(os.path.join(stack_group_dir_c, config_filename), 'w') as\
+                    config_file:
+                yaml.safe_dump(
+                    config_c, stream=config_file, default_flow_style=False
+                )
+
+            self.context.project_path = project_path
+            reader = ConfigReader(self.context)
+
+            config_a = reader.read("A/config.yaml")
+
+            assert config_a == {
+                "project_path": project_path,
+                "stack_group_path": "A",
+                "keyA": "A",
+                "shared": "A"
+            }
+
+            config_b = reader.read("A/B/config.yaml")
+
+            assert config_b == {
+                "project_path": project_path,
+                "stack_group_path": "A/B",
+                "keyA": "A",
+                "keyB": "B",
+                "shared": "B",
+                "parent": "A"
+            }
+
+            config_c = reader.read(
+                "A/B/C/config.yaml"
+            )
+
+            assert config_c == {
+                "project_path": project_path,
+                "stack_group_path": "A/B/C",
+                "keyA": "A",
+                "keyB": "B",
+                "keyC": "C",
+                "shared": "C",
+                "parent": "B"
+            }
+
     def test_read_reads_config_file_with_base_config(self):
         with self.runner.isolated_filesystem():
             project_path = os.path.abspath('./example')