Merge pull request #576 from remind101/graph-command

Add "graph" subcommand

stacker/actions/graph.py

@@ -0,0 +1,75 @@
+import logging
+import sys
+import json
+
+from .base import BaseAction, plan
+
+
+logger = logging.getLogger(__name__)
+
+
+def each_step(graph):
+    """Returns an iterator that yields each step and it's direct
+    dependencies.
+    """
+
+    steps = graph.topological_sort()
+    steps.reverse()
+
+    for step in steps:
+        deps = graph.downstream(step.name)
+        yield (step, deps)
+
+
+def dot_format(out, graph, name="digraph"):
+    """Outputs the graph using the graphviz "dot" format."""
+
+    out.write("digraph %s {\n" % name)
+    for step, deps in each_step(graph):
+        for dep in deps:
+            out.write("  \"%s\" -> \"%s\";\n" % (step, dep))
+
+    out.write("}\n")
+
+
+def json_format(out, graph):
+    """Outputs the graph in a machine readable JSON format."""
+    steps = {}
+    for step, deps in each_step(graph):
+        steps[step.name] = {}
+        steps[step.name]["deps"] = [dep.name for dep in deps]
+
+    json.dump({"steps": steps}, out, indent=4)
+    out.write("\n")
+
+
+FORMATTERS = {
+    "dot": dot_format,
+    "json": json_format,
+}
+
+
+class Action(BaseAction):
+
+    def _generate_plan(self):
+        return plan(
+            description="Print graph",
+            action=None,
+            stacks=self.context.get_stacks(),
+            targets=self.context.stack_names)
+
+    def run(self, format=None, reduce=False, *args, **kwargs):
+        """Generates the underlying graph and prints it.
+
+        """
+        plan = self._generate_plan()
+        if reduce:
+            # This will performa a transitive reduction on the underlying
+            # graph, producing less edges. Mostly useful for the "dot" format,
+            # when converting to PNG, so it creates a prettier/cleaner
+            # dependency graph.
+            plan.graph.transitive_reduction()
+
+        fn = FORMATTERS[format]
+        fn(sys.stdout, plan.graph)
+        sys.stdout.flush()

stacker/commands/stacker/__init__.py

@@ -4,6 +4,7 @@
 from .destroy import Destroy
 from .info import Info
 from .diff import Diff
+from .graph import Graph
 from .base import BaseCommand
 from ...config import render_parse_load as load_config
 from ...context import Context
@@ -17,7 +18,7 @@
 class Stacker(BaseCommand):
 
     name = "stacker"
-    subcommands = (Build, Destroy, Info, Diff)
+    subcommands = (Build, Destroy, Info, Diff, Graph)
 
     def configure(self, options, **kwargs):
         super(Stacker, self).configure(options, **kwargs)

stacker/commands/stacker/graph.py

@@ -0,0 +1,32 @@
+"""Prints the the relationships between steps as a graph.
+
+"""
+
+from .base import BaseCommand
+from ...actions import graph
+
+
+class Graph(BaseCommand):
+
+    name = "graph"
+    description = __doc__
+
+    def add_arguments(self, parser):
+        super(Graph, self).add_arguments(parser)
+        parser.add_argument("-f", "--format", default="dot",
+                            choices=list(graph.FORMATTERS.iterkeys()),
+                            help="The format to print the graph in.")
+        parser.add_argument("--reduce", action="store_true",
+                            help="When provided, this will create a "
+                                 "graph with less edges, by performing "
+                                 "a transitive reduction on the underlying "
+                                 "graph. While this will produce a less "
+                                 "noisy graph, it is slower.")
+
+    def run(self, options, **kwargs):
+        super(Graph, self).run(options, **kwargs)
+        action = graph.Action(options.context,
+                              provider_builder=options.provider_builder)
+        action.execute(
+            format=options.format,
+            reduce=options.reduce)

stacker/dag/__init__.py

@@ -161,6 +161,30 @@ def walk(self, walk_func):
         for n in nodes:
             walk_func(n)
 
+    def transitive_reduction(self):
+        """ Performs a transitive reduction on the DAG. The transitive
+        reduction of a graph is a graph with as few edges as possible with the
+        same reachability as the original graph.
+
+        See https://en.wikipedia.org/wiki/Transitive_reduction
+        """
+
+        graph = self.graph
+        for x in graph.keys():
+            for y in graph.keys():
+                for z in graph.keys():
+                    # Edge from x -> y
+                    xy = y in graph[x]
+                    # Edge from y -> x
+                    yz = z in graph[y]
+                    # Edge from x -> z
+                    xz = z in graph[x]
+
+                    # If edges xy and yz exist, remove edge xz.
+                    if xz and xy and yz:
+                        logger.debug("Removing edge %s -> %s" % (x, z))
+                        graph[x].remove(z)
+
     def rename_edges(self, old_node_name, new_node_name):
         """ Change references to a node in existing edges.
 

stacker/plan.py

@@ -248,6 +248,9 @@ def connect(self, step, dep):
         except DAGValidationError as e:
             raise GraphError(e, step.name, dep)
 
+    def transitive_reduction(self):
+        self.dag.transitive_reduction()
+
     def walk(self, walker, walk_func):
         def fn(step_name):
             step = self.steps[step_name]

stacker/tests/test_dag.py

@@ -164,6 +164,38 @@ def test_size():
     assert dag.size() == 3
 
 
+@with_setup(blank_setup)
+def test_transitive_reduction_no_reduction():
+    dag = DAG()
+    dag.from_dict({'a': ['b', 'c'],
+                   'b': ['d'],
+                   'c': ['d'],
+                   'd': []})
+    dag.transitive_reduction()
+    assert dag.graph == {'a': set(['b', 'c']),
+                         'b': set('d'),
+                         'c': set('d'),
+                         'd': set()}
+
+
+@with_setup(blank_setup)
+def test_transitive_reduction():
+    dag = DAG()
+    # https://en.wikipedia.org/wiki/Transitive_reduction#/media/File:Tred-G.svg
+    dag.from_dict({'a': ['b', 'c', 'd', 'e'],
+                   'b': ['d'],
+                   'c': ['d', 'e'],
+                   'd': ['e'],
+                   'e': []})
+    dag.transitive_reduction()
+    # https://en.wikipedia.org/wiki/Transitive_reduction#/media/File:Tred-Gprime.svg
+    assert dag.graph == {'a': set(['b', 'c']),
+                         'b': set('d'),
+                         'c': set('d'),
+                         'd': set('e'),
+                         'e': set()}
+
+
 @with_setup(blank_setup)
 def test_threaded_walker():
     dag = DAG()

tests/suite.bats

@@ -71,6 +71,105 @@ EOF
   assert_has_line "Duplicate stack vpc found"
 }
 
+@test "stacker graph - json format" {
+  config() {
+    cat <<EOF
+namespace: ${STACKER_NAMESPACE}
+stacks:
+  - name: vpc
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy
+  - name: bastion1
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy2
+    variables:
+      StringVariable: \${output vpc::DummyId}
+  - name: bastion2
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy
+    variables:
+      StringVariable: \${output vpc::DummyId}
+  - name: app1
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy2
+    variables:
+      StringVariable: \${output bastion1::DummyId}
+  - name: app2
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy
+    variables:
+      StringVariable: \${output bastion2::DummyId}
+EOF
+  }
+
+  # Print the graph
+  stacker graph -f json <(config)
+  assert "$status" -eq 0
+  cat <<-EOF | diff -y <(echo "$output" | grep -v "Using default") -
+{
+    "steps": {
+        "app1": {
+            "deps": [
+                "bastion1"
+            ]
+        }, 
+        "app2": {
+            "deps": [
+                "bastion2"
+            ]
+        }, 
+        "bastion2": {
+            "deps": [
+                "vpc"
+            ]
+        }, 
+        "vpc": {
+            "deps": []
+        }, 
+        "bastion1": {
+            "deps": [
+                "vpc"
+            ]
+        }
+    }
+}
+EOF
+}
+
+@test "stacker graph - dot format" {
+  config() {
+    cat <<EOF
+namespace: ${STACKER_NAMESPACE}
+stacks:
+  - name: vpc
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy
+  - name: bastion1
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy2
+    variables:
+      StringVariable: \${output vpc::DummyId}
+  - name: bastion2
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy
+    variables:
+      StringVariable: \${output vpc::DummyId}
+  - name: app1
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy2
+    variables:
+      StringVariable: \${output bastion1::DummyId}
+  - name: app2
+    class_path: stacker.tests.fixtures.mock_blueprints.Dummy
+    variables:
+      StringVariable: \${output bastion2::DummyId}
+EOF
+  }
+
+  # Print the graph
+  stacker graph -f dot <(config)
+  assert "$status" -eq 0
+  cat <<-EOF | diff -y <(echo "$output" | grep -v "Using default") -
+digraph digraph {
+  "bastion2" -> "vpc";
+  "bastion1" -> "vpc";
+  "app2" -> "bastion2";
+  "app1" -> "bastion1";
+}
+EOF
+}
+
 @test "stacker build - missing variable" {
   needs_aws