Merge pull request #572 from cloudtools/targets

Targets

docs/config.rst

@@ -371,6 +371,9 @@ A stack has the following keys:
   (optional) a list of other stacks this stack requires. This is for explicit
   dependencies - you do not need to set this if you refer to another stack in
   a Parameter, so this is rarely necessary.
+**required_by:**
+  (optional) a list of other stacks or targets that require this stack. It's an
+  inverse to ``requires``.
 **tags:**
   (optional) a dictionary of CloudFormation tags to apply to this stack. This
   will be combined with the global tags, but these tags will take precendence.
@@ -423,6 +426,37 @@ Here's an example from stacker_blueprints_, used to create a VPC::
           - 10.128.20.0/22
         CidrBlock: 10.128.0.0/16
 
+Targets
+-------
+
+In stacker, **targets** can be used as a lightweight method to group a number
+of stacks together, as a named "target" in the graph. Internally, this adds a
+node to the underlying DAG, which can then be used alongside the `--targets`
+flag. If you're familiar with the concept of "targets" in systemd, the concept
+is the same.
+
+**name:**
+  The logical name for this target.
+**requires:**
+  (optional) a list of stacks or other targets this target requires.
+**required_by:**
+  (optional) a list of stacks or other targets that require this target.
+
+Here's an example of a target that will execute all "database" stacks::
+
+  targets:
+    - name: databases
+
+  stacks:
+    - name: dbA
+      class_path: blueprints.DB
+      required_by:
+        - databases
+    - name: dbB
+      class_path: blueprints.DB
+      required_by:
+        - databases
+
 Variables
 ==========
 

stacker/actions/base.py

@@ -8,12 +8,16 @@
 import threading
 
 from ..dag import walk, ThreadedWalker
-from ..plan import Step, build_plan
+from ..plan import Step, build_plan, build_graph
 
 import botocore.exceptions
 from stacker.session_cache import get_session
 from stacker.exceptions import PlanFailed
 
+from ..status import (
+    COMPLETE
+)
+
 from stacker.util import (
     ensure_s3_bucket,
     get_s3_endpoint,
@@ -52,16 +56,15 @@ def build_walker(concurrency):
     return ThreadedWalker(concurrency).walk
 
 
-def plan(description, action, stacks,
-         targets=None, tail=None,
-         reverse=False):
+def plan(description, stack_action, context,
+         tail=None, reverse=False):
     """A simple helper that builds a graph based plan from a set of stacks.
 
     Args:
         description (str): a description of the plan.
         action (func): a function to call for each stack.
-        stacks (list): a list of :class:`stacker.stack.Stack` objects to build.
-        targets (list): an optional list of targets to filter the graph to.
+        context (:class:`stacker.context.Context`): a
+            :class:`stacker.context.Context` to build the plan from.
         tail (func): an optional function to call to tail the stack progress.
         reverse (bool): if True, execute the graph in reverse (useful for
             destroy actions).
@@ -70,14 +73,22 @@ def plan(description, action, stacks,
         :class:`plan.Plan`: The resulting plan object
     """
 
+    def target_fn(*args, **kwargs):
+        return COMPLETE
+
     steps = [
-        Step(stack, fn=action, watch_func=tail)
-        for stack in stacks]
+        Step(stack, fn=stack_action, watch_func=tail)
+        for stack in context.get_stacks()]
+
+    steps += [
+        Step(target, fn=target_fn) for target in context.get_targets()]
+
+    graph = build_graph(steps)
 
     return build_plan(
         description=description,
-        steps=steps,
-        targets=targets,
+        graph=graph,
+        targets=context.stack_names,
         reverse=reverse)
 
 

stacker/actions/build.py

@@ -386,10 +386,9 @@ def _stack_policy(self, stack):
     def _generate_plan(self, tail=False):
         return plan(
             description="Create/Update stacks",
-            action=self._launch_stack,
+            stack_action=self._launch_stack,
             tail=self._tail_stack if tail else None,
-            stacks=self.context.get_stacks(),
-            targets=self.context.stack_names)
+            context=self.context)
 
     def pre_run(self, outline=False, dump=False, *args, **kwargs):
         """Any steps that need to be taken prior to running the action."""

stacker/actions/destroy.py

@@ -38,10 +38,9 @@ class Action(BaseAction):
     def _generate_plan(self, tail=False):
         return plan(
             description="Destroy stacks",
-            action=self._destroy_stack,
+            stack_action=self._destroy_stack,
             tail=self._tail_stack if tail else None,
-            stacks=self.context.get_stacks(),
-            targets=self.context.stack_names,
+            context=self.context,
             reverse=True)
 
     def _destroy_stack(self, stack, **kwargs):

stacker/actions/diff.py

@@ -271,9 +271,8 @@ def _diff_stack(self, stack, **kwargs):
     def _generate_plan(self):
         return plan(
             description="Diff stacks",
-            action=self._diff_stack,
-            stacks=self.context.get_stacks(),
-            targets=self.context.stack_names)
+            stack_action=self._diff_stack,
+            context=self.context)
 
     def run(self, concurrency=0, *args, **kwargs):
         plan = self._generate_plan()

stacker/actions/graph.py

@@ -57,9 +57,8 @@ class Action(BaseAction):
     def _generate_plan(self):
         return plan(
             description="Print graph",
-            action=None,
-            stacks=self.context.get_stacks(),
-            targets=self.context.stack_names)
+            stack_action=None,
+            context=self.context)
 
     def run(self, format=None, reduce=False, *args, **kwargs):
         """Generates the underlying graph and prints it.

stacker/commands/stacker/build.py

@@ -28,7 +28,7 @@ def add_arguments(self, parser):
                             help="If a stackname is provided to --force, it "
                                  "will be updated, even if it is locked in "
                                  "the config.")
-        parser.add_argument("--stacks", action="append",
+        parser.add_argument("--targets", "--stacks", action="append",
                             metavar="STACKNAME", type=str,
                             help="Only work on the stacks given, and their "
                                  "dependencies. Can be specified more than "
@@ -58,4 +58,4 @@ def run(self, options, **kwargs):
                        dump=options.dump)
 
     def get_context_kwargs(self, options, **kwargs):
-        return {"stack_names": options.stacks, "force_stacks": options.force}
+        return {"stack_names": options.targets, "force_stacks": options.force}

stacker/commands/stacker/destroy.py

@@ -22,7 +22,7 @@ def add_arguments(self, parser):
         parser.add_argument("-f", "--force", action="store_true",
                             help="Whether or not you want to go through "
                                  " with destroying the stacks")
-        parser.add_argument("--stacks", action="append",
+        parser.add_argument("--targets", "--stacks", action="append",
                             metavar="STACKNAME", type=str,
                             help="Only work on the stacks given. Can be "
                                  "specified more than once. If not specified "
@@ -48,4 +48,4 @@ def run(self, options, **kwargs):
                        tail=options.tail)
 
     def get_context_kwargs(self, options, **kwargs):
-        return {"stack_names": options.stacks}
+        return {"stack_names": options.targets}

stacker/config/__init__.py

@@ -281,6 +281,14 @@ class Hook(Model):
     args = DictType(AnyType)
 
 
+class Target(Model):
+    name = StringType(required=True)
+
+    requires = ListType(StringType, serialize_when_none=False)
+
+    required_by = ListType(StringType, serialize_when_none=False)
+
+
 class Stack(Model):
     name = StringType(required=True)
 
@@ -298,6 +306,8 @@ class Stack(Model):
 
     requires = ListType(StringType, serialize_when_none=False)
 
+    required_by = ListType(StringType, serialize_when_none=False)
+
     locked = BooleanType(default=False)
 
     enabled = BooleanType(default=True)
@@ -405,6 +415,9 @@ class Config(Model):
 
     lookups = DictType(StringType, serialize_when_none=False)
 
+    targets = ListType(
+        ModelType(Target), serialize_when_none=False)
+
     stacks = ListType(
         ModelType(Stack), default=[])
 

stacker/context.py

@@ -7,6 +7,7 @@
 
 from stacker.config import Config
 from .stack import Stack
+from .target import Target
 
 logger = logging.getLogger(__name__)
 
@@ -123,6 +124,21 @@ def mappings(self):
     def _get_stack_definitions(self):
         return self.config.stacks
 
+    def get_targets(self):
+        """Returns the named targets that are specified in the config.
+
+        Returns:
+            list: a list of :class:`stacker.target.Target` objects
+
+        """
+        if not hasattr(self, "_targets"):
+            targets = []
+            for target_def in self.config.targets or []:
+                target = Target(target_def)
+                targets.append(target)
+            self._targets = targets
+        return self._targets
+
     def get_stacks(self):
         """Get the stacks for the current action.
 

stacker/plan.py

@@ -60,7 +60,7 @@ def __init__(self, stack, fn, watch_func=None):
         self.watch_func = watch_func
 
     def __repr__(self):
-        return "<stacker.plan.Step:%s>" % (self.stack.fqn,)
+        return "<stacker.plan.Step:%s>" % (self.stack.name,)
 
     def __str__(self):
         return self.stack.name
@@ -105,6 +105,10 @@ def name(self):
     def requires(self):
         return self.stack.requires
 
+    @property
+    def required_by(self):
+        return self.stack.required_by
+
     @property
     def completed(self):
         """Returns True if the step is in a COMPLETE state."""
@@ -147,7 +151,8 @@ def set_status(self, status):
                          status.name)
             self.status = status
             self.last_updated = time.time()
-            log_step(self)
+            if self.stack.logging:
+                log_step(self)
 
     def complete(self):
         """A shortcut for set_status(COMPLETE)"""
@@ -162,21 +167,20 @@ def submit(self):
         self.set_status(SUBMITTED)
 
 
-def build_plan(description, steps,
+def build_plan(description, graph,
                targets=None, reverse=False):
     """Builds a plan from a list of steps.
     Args:
         description (str): an arbitrary string to
             describe the plan.
-        steps (list): a list of :class:`Step` objects to execute.
+        graph (:class:`Graph`): a list of :class:`Graph` to execute.
         targets (list): an optional list of step names to filter the graph to.
             If provided, only these steps, and their transitive dependencies
             will be executed. If no targets are specified, every node in the
             graph will be executed.
         reverse (bool): If provided, the graph will be walked in reverse order
             (dependencies last).
     """
-    graph = build_graph(steps)
 
     # If we want to execute the plan in reverse (e.g. Destroy), transpose the
     # graph.
@@ -187,7 +191,7 @@ def build_plan(description, steps,
     if targets:
         nodes = []
         for target in targets:
-            for step in steps:
+            for k, step in graph.steps.items():
                 if step.name == target:
                     nodes.append(step.name)
         graph = graph.filtered(nodes)
@@ -208,7 +212,10 @@ def build_graph(steps):
 
     for step in steps:
         for dep in step.requires:
-            graph.connect(step, dep)
+            graph.connect(step.name, dep)
+
+        for parent in step.required_by:
+            graph.connect(parent, step.name)
 
     return graph
 
@@ -246,11 +253,11 @@ def add_step(self, step):
 
     def connect(self, step, dep):
         try:
-            self.dag.add_edge(step.name, dep)
+            self.dag.add_edge(step, dep)
         except KeyError as e:
-            raise GraphError(e, step.name, dep)
+            raise GraphError(e, step, dep)
         except DAGValidationError as e:
-            raise GraphError(e, step.name, dep)
+            raise GraphError(e, step, dep)
 
     def transitive_reduction(self):
         self.dag.transitive_reduction()

stacker/stack.py

@@ -65,6 +65,7 @@ class Stack(object):
 
     def __init__(self, definition, context, variables=None, mappings=None,
                  locked=False, force=False, enabled=True, protected=False):
+        self.logging = True
         self.name = definition.name
         self.fqn = context.get_fqn(definition.stack_name or self.name)
         self.region = definition.region
@@ -82,6 +83,10 @@ def __init__(self, definition, context, variables=None, mappings=None,
     def __repr__(self):
         return self.fqn
 
+    @property
+    def required_by(self):
+        return self.definition.required_by or []
+
     @property
     def requires(self):
         requires = set(self.definition.requires or [])

stacker/target.py

@@ -0,0 +1,16 @@
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+
+
+class Target(object):
+    """A "target" is just a node in the stacker graph that does nothing, except
+    specify dependencies. These can be useful as a means of logically grouping
+    a set of stacks together that can be targeted with the `--targets` flag.
+    """
+
+    def __init__(self, definition):
+        self.name = definition.name
+        self.requires = definition.requires or []
+        self.required_by = definition.required_by or []
+        self.logging = False