[Resolve # 683] Introducing the Diff Command (#1132)

Running `sceptre diff` generates a difference between:
* The Stack config and template that is _currently deployed_ (if it is), and
* The Stack config and template locally that _would_ be deployed, if deployment were triggered now.

There are two different supported types of diff output:
* deepdiff (which indicates only the added/removed/changed keys and values from both config and template)
* difflib (which produces a more "traditional" diff output between the configs and templates)

docs/_source/apidoc/sceptre.cli.rst

@@ -25,6 +25,14 @@ sceptre.cli.delete module
     :undoc-members:
     :show-inheritance:
 
+sceptre.cli.diff module
+_______________________
+
+.. automodule:: sceptre.cli.diff
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 sceptre.cli.describe module
 ---------------------------
 

docs/_source/apidoc/sceptre.diffing.rst

@@ -0,0 +1,26 @@
+sceptre.diffing package
+=======================
+
+.. automodule:: sceptre.diffing
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Submodules
+----------
+
+sceptre.diffing.stack_differ module
+-----------------------------------
+
+.. automodule:: sceptre.diffing.stack_differ
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+sceptre.diffing.diff_writer module
+----------------------------------
+
+.. automodule:: sceptre.diffing.diff_writer
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/_source/apidoc/sceptre.rst

@@ -13,6 +13,7 @@ Subpackages
 
     sceptre.cli
     sceptre.config
+    sceptre.diffing
     sceptre.hooks
     sceptre.plan
     sceptre.resolvers

docs/_source/conf.py

@@ -189,6 +189,10 @@
         'https://boto3.readthedocs.io/en/latest/',
         'https://boto3.readthedocs.io/en/latest/objects.inv',
     ),
+    'deepdiff': (
+        'https://zepworks.com/deepdiff/current/',
+        None
+    )
 }
 
 # other configuration
@@ -197,5 +201,8 @@
 
 nitpick_ignore = [
     ('py:class', 'json.encoder.JSONEncoder'),
-    ('py:class', 'sceptre.config.reader.Attributes')
+    ('py:class', 'sceptre.config.reader.Attributes'),
+    ('py:class', 'sceptre.diffing.stack_differ.DiffType'),
+    ('py:obj', 'sceptre.diffing.stack_differ.DiffType'),
+    ('py:class', 'TextIO')
 ]

integration-tests/features/stack-diff.feature

@@ -0,0 +1,80 @@
+Feature: Stack Diff
+  Scenario Outline: Diff on stack that exists with no changes
+    Given stack "1/A" exists in "CREATE_COMPLETE" state
+    And the template for stack "1/A" is "valid_template.json"
+    When the user diffs stack "1/A" with "<diff_type>"
+    Then a diff is returned with no "template" difference
+    And a diff is returned with no "config" difference
+    And a diff is returned with "is_deployed" = "True"
+
+    Examples: DiffTypes
+      | diff_type |
+      | deepdiff  |
+      | difflib   |
+
+  Scenario Outline: Diff on stack that doesnt exist
+    Given stack "1/A" does not exist
+    And the template for stack "1/A" is "valid_template.json"
+    When the user diffs stack "1/A" with "<diff_type>"
+    Then a diff is returned with "is_deployed" = "False"
+    And a diff is returned with a "template" difference
+    And a diff is returned with a "config" difference
+
+    Examples: DiffTypes
+      | diff_type |
+      | deepdiff  |
+      | difflib   |
+
+  Scenario Outline: Diff on stack that exists in non-deployed state
+    Given stack "1/A" exists in "<status>" state
+    And the template for stack "1/A" is "valid_template.json"
+    When the user diffs stack "1/A" with "<diff_type>"
+    Then a diff is returned with a "template" difference
+    And a diff is returned with a "config" difference
+    And a diff is returned with "is_deployed" = "False"
+
+    Examples: DeepDiff
+      | diff_type | status            |
+      | deepdiff  | CREATE_FAILED     |
+      | deepdiff  | ROLLBACK_COMPLETE |
+      | difflib   | CREATE_FAILED     |
+      | difflib   | ROLLBACK_COMPLETE |
+
+  Scenario Outline: Diff on stack with only template changes
+    Given stack "1/A" exists in "CREATE_COMPLETE" state
+    And the template for stack "1/A" is "updated_template.json"
+    When the user diffs stack "1/A" with "<diff_type>"
+    Then a diff is returned with a "template" difference
+    And a diff is returned with no "config" difference
+
+    Examples: DiffTypes
+      | diff_type |
+      | deepdiff  |
+      | difflib   |
+
+  Scenario Outline: Diff on stack with only configuration changes
+    Given stack "1/A" exists in "CREATE_COMPLETE" state
+    And the template for stack "1/A" is "valid_template.json"
+    And the stack config for stack "1/A" has changed
+    When the user diffs stack "1/A" with "<diff_type>"
+    Then a diff is returned with a "config" difference
+    And a diff is returned with no "template" difference
+
+    Examples: DiffTypes
+      | diff_type |
+      | deepdiff  |
+      | difflib   |
+
+
+  Scenario Outline: Diff on stack with both configuration and template changes
+    Given stack "1/A" exists in "CREATE_COMPLETE" state
+    And the template for stack "1/A" is "updated_template.json"
+    And the stack config for stack "1/A" has changed
+    When the user diffs stack "1/A" with "<diff_type>"
+    Then a diff is returned with a "config" difference
+    And a diff is returned with a "template" difference
+
+    Examples: DiffTypes
+      | diff_type |
+      | deepdiff  |
+      | difflib   |

integration-tests/steps/stacks.py

@@ -1,3 +1,9 @@
+from ast import literal_eval
+from copy import deepcopy
+from io import StringIO
+from pathlib import Path
+from typing import Type
+
 from behave import *
 import time
 import os
@@ -7,6 +13,8 @@
 from botocore.exceptions import ClientError
 from helpers import read_template_file, get_cloudformation_stack_name
 from helpers import retry_boto_call
+from sceptre.diffing.diff_writer import DeepDiffWriter, DiffWriter
+from sceptre.diffing.stack_differ import DeepDiffStackDiffer, DifflibStackDiffer, StackDiff
 
 from sceptre.plan.plan import SceptrePlan
 from sceptre.context import SceptreContext
@@ -112,6 +120,31 @@ def step_impl(context, dependant_stack_name, stack_name):
     assert False
 
 
+@given('the stack config for stack "{stack_name}" has changed')
+def step_impl(context, stack_name):
+    sceptre_context = SceptreContext(
+        command_path=stack_name + '.yaml',
+        project_path=context.sceptre_dir,
+        ignore_dependencies=True
+    )
+    yaml_file = Path(sceptre_context.full_config_path()) / f'{stack_name}.yaml'
+    with yaml_file.open(mode='r') as f:
+        loaded = yaml.load(f)
+
+    original_config = deepcopy(loaded)
+    loaded['stack_tags'] = {
+        'NewTag': 'NewValue'
+    }
+    dump_stack_config(yaml_file, loaded)
+
+    context.add_cleanup(dump_stack_config, yaml_file, original_config)
+
+
+def dump_stack_config(config_path: Path, config_dict: dict):
+    with config_path.open(mode='w') as f:
+        yaml.safe_dump(config_dict, f)
+
+
 @when('the user creates stack "{stack_name}"')
 def step_impl(context, stack_name):
     sceptre_context = SceptreContext(
@@ -285,6 +318,27 @@ def step_impl(context, stack_name):
     context.output = list(sceptre_plan.describe_resources().values())
 
 
+@when('the user diffs stack "{stack_name}" with "{diff_type}"')
+def step_impl(context, stack_name, diff_type):
+    sceptre_context = SceptreContext(
+        command_path=stack_name + '.yaml',
+        project_path=context.sceptre_dir
+    )
+    sceptre_plan = SceptrePlan(sceptre_context)
+    differ_classes = {
+        'deepdiff': DeepDiffStackDiffer,
+        'difflib': DifflibStackDiffer
+    }
+    writer_class = {
+        'deepdiff': DeepDiffWriter,
+        'difflib': DeepDiffWriter
+    }
+
+    differ = differ_classes[diff_type]()
+    context.writer_class = writer_class[diff_type]
+    context.output = list(sceptre_plan.diff(differ).values())
+
+
 @then(
     'stack "{stack_name}" in "{region_name}" '
     'exists in "{desired_status}" state'
@@ -362,6 +416,32 @@ def step_impl(context, stack_name, dependant_stack_name, desired_state):
     assert (dep_status == desired_state)
 
 
+@then('a diff is returned with "{attribute}" = "{value}"')
+def step_impl(context, attribute, value):
+    for diff in context.output:
+        expected_value = literal_eval(value)
+        actual_value = getattr(diff, attribute)
+        assert actual_value == expected_value
+
+
+@then('a diff is returned with {a_or_no} "{kind}" difference')
+def step_impl(context, a_or_no, kind):
+    if a_or_no == 'a':
+        test_value = True
+    elif a_or_no == 'no':
+        test_value = False
+    else:
+        raise ValueError('Only "a" or "no" accepted in this condition')
+
+    writer_class: Type[DiffWriter] = context.writer_class
+    difference_property = f'has_{kind}_difference'
+
+    for diff in context.output:
+        diff: StackDiff
+        writer = writer_class(diff, StringIO(), 'yaml')
+        assert getattr(writer, difference_property) is test_value
+
+
 def get_stack_tags(context, stack_name, region_name=None):
     if region_name is not None:
         stack = boto3.resource('cloudformation', region_name=region_name).Stack

requirements/prod.txt

@@ -1,6 +1,8 @@
 boto3>=1.3.0,<2
 click>=7.0,<9.0
 colorama>=0.2.5,<0.4.4
+cfn-flip>=1.2.3<2.0
+deepdiff>=5.5.0,<6.0
 Jinja2>=2.8,<3
 jsonschema>=3.2,<3.3
 networkx>=2.4,<2.6

sceptre/cli/__init__.py

@@ -19,13 +19,15 @@
 from sceptre.cli.update import update_command
 from sceptre.cli.delete import delete_command
 from sceptre.cli.launch import launch_command
+from sceptre.cli.diff import diff_command
 from sceptre.cli.execute import execute_command
 from sceptre.cli.describe import describe_group
 from sceptre.cli.list import list_group
 from sceptre.cli.policy import set_policy_command
 from sceptre.cli.status import status_command
 from sceptre.cli.template import (validate_command, generate_command,
-                                  estimate_cost_command)
+                                  estimate_cost_command,
+                                  fetch_remote_template_command)
 from sceptre.cli.helpers import catch_exceptions, setup_vars
 
 
@@ -82,3 +84,5 @@ def cli(
 cli.add_command(status_command)
 cli.add_command(list_group)
 cli.add_command(describe_group)
+cli.add_command(fetch_remote_template_command)
+cli.add_command(diff_command)