adds --with-dependencies flag (#650)

Adds ability to run an action and all of its dependencies (up to that action)

fixes #625

ci/test/actions-demo

@@ -38,9 +38,122 @@ cat <<EOF > main.workflow
 workflow "clone" {
   resolves = "test"
 }
+
+action "dependency four" {
+  uses = "docker://busybox"
+  runs = ["ls", "-ltr"]
+}
+
+action "dependency three" {
+  uses = "docker://busybox"
+  runs = ["ls", "-ltr"]
+}
+
+action "dependency two" {
+  needs = ["dependency three", "dependency four"]
+  uses = "docker://busybox"
+  runs = ["ls", "-ltr"]
+}
+
+action "dependency one" {
+  needs = ["dependency two"]
+  uses = "docker://busybox"
+  runs = ["ls", "-ltr"]
+}
+
 action "test" {
+  needs = ["dependency one"]
   uses = "https://gitlab.com/barbaragd/action-test@master"
 }
 EOF
 
-popper run
+# popper run
+
+# test --with-dependencies flag
+popper run --dry-run test
+popper run --dry-run --with-dependencies test
+popper run --dry-run --with-dependencies 'dependency one'
+
+
+cat <<EOF > main.workflow
+workflow "wf" {
+  resolves = "d"
+}
+
+action "a1" {
+  uses = "sh"
+  args = "ls"
+}
+
+action "a2" {
+  uses = "sh"
+  args = "ls"
+}
+
+action "b" {
+  needs = ["a1", "a2"]
+  uses = "sh"
+  args = "ls"
+}
+
+action "c" {
+  needs = ["a1", "a2"]
+  uses = "sh"
+  args = "ls"
+}
+
+action "d" {
+  needs = ["b", "c"]
+  uses = "sh"
+  args = "ls"
+}
+EOF
+
+! popper run --with-dependencies
+popper run --with-dependencies a1
+popper run --with-dependencies a2
+popper run --with-dependencies b
+popper run --with-dependencies c
+popper run --with-dependencies d
+
+cat <<EOF > main.workflow
+workflow "wf" {
+  resolves = "d"
+}
+
+action "a1" {
+  uses = "sh"
+  args = "ls"
+}
+
+action "a2" {
+  uses = "sh"
+  args = "ls"
+}
+
+action "b" {
+  needs = ["a1", "a2"]
+  uses = "sh"
+  args = "ls"
+}
+
+action "c" {
+  needs = ["a1", "a2"]
+  uses = "sh"
+  args = "ls"
+}
+
+action "d" {
+  needs = ["a2"]
+  uses = "sh"
+  args = "ls"
+}
+EOF
+
+popper run a1
+popper run a2
+popper run b
+popper run c
+popper run d
+popper run --with-dependencies b
+popper run --with-dependencies d

cli/popper/commands/cmd_run.py

@@ -14,6 +14,12 @@
     'run', short_help='Run a workflow or action.')
 @click.argument(
     'action', required=False)
+@click.option(
+    '--with-dependencies',
+    help='Run the action with all its dependencies.',
+    required=False,
+    is_flag=True
+)
 @click.option(
     '--workspace',
     help='Path to workspace folder.',
@@ -81,7 +87,8 @@
 )
 @pass_context
 def cli(ctx, action, wfile, skip, workspace, reuse,
-        recursive, quiet, debug, dry_run, parallel, log_file):
+        recursive, quiet, debug, dry_run, parallel,
+        log_file, with_dependencies):
     """Executes one or more pipelines and reports on their status.
     """
     popper.scm.get_git_root_folder()
@@ -100,14 +107,14 @@ def cli(ctx, action, wfile, skip, workspace, reuse,
         for wfile in wfile_list:
             log.info("Found and running workflow at " + wfile)
             run_pipeline(action, wfile, skip, workspace, reuse,
-                         dry_run, parallel)
+                         dry_run, parallel, with_dependencies)
     else:
         run_pipeline(action, wfile, skip, workspace, reuse,
-                     dry_run, parallel)
+                     dry_run, parallel, with_dependencies)
 
 
 def run_pipeline(action, wfile, skip, workspace, reuse,
-                 dry_run, parallel):
+                 dry_run, parallel, with_dependencies):
 
     # Initialize a Worklow. During initialization all the validation
     # takes place automatically.
@@ -127,7 +134,8 @@ def run_pipeline(action, wfile, skip, workspace, reuse,
         log.warn("Using --parallel may result in interleaved output. "
                  "You may use --quiet flag to avoid confusion.")
 
-    pipeline.run(action, skip, workspace, reuse, dry_run, parallel)
+    pipeline.run(action, skip, workspace, reuse, dry_run,
+                 parallel, with_dependencies)
 
     if action:
         log.info('Action "{}" finished successfully.'.format(action))

cli/popper/gha.py

@@ -148,7 +148,7 @@ def instantiate_runners(self, wf, workspace, env, dry_run):
                     dry_run)
 
     def run(self, action, skip, workspace, reuse, dry_run,
-            parallel, skip_secrets_prompt=False):
+            parallel, with_dependencies, skip_secrets_prompt=False):
         """Run the pipeline or a specific action"""
         os.environ['WORKSPACE'] = workspace
 
@@ -157,8 +157,12 @@ def run(self, action, skip, workspace, reuse, dry_run,
         else:
             repo_id = 'unknown'
 
+        if with_dependencies and (not action):
+            log.fail('`--with-dependencies` can be used only with '
+                     'action argument.')
+
         if skip and action:
-            log.fail('`--skip` cant be used when `action` argument'
+            log.fail('`--skip` can\'t be used when action argument '
                      'is passed.')
 
         new_wf = deepcopy(self.wf)
@@ -167,7 +171,7 @@ def run(self, action, skip, workspace, reuse, dry_run,
             new_wf = self.wf.skip_actions(skip)
 
         if action:
-            new_wf = self.wf.filter_action(action)
+            new_wf = self.wf.filter_action(action, with_dependencies)
 
         new_wf.check_for_unreachable_actions(skip)
 

cli/popper/parser.py

@@ -324,7 +324,7 @@ def skip_actions(self, skip_list):
         workflow.props['skip_list'] = list(skip_list)
         return workflow
 
-    def filter_action(self, action):
+    def filter_action(self, action, with_dependencies):
         """Filters out all actions except the one passed in
         the argument from the workflow.
 
@@ -334,20 +334,53 @@ def filter_action(self, action):
         Returns:
             Workflow : The updated workflow object.
         """
+        # Recursively generate root when an action is run
+        # with the `--with-dependencies` flag.
+        def find_root_recursively(workflow, action, required_actions):
+            required_actions.add(action)
+            if workflow.get_action(action).get('needs', None):
+                for a in workflow.get_action(action)['needs']:
+                    find_root_recursively(workflow, a, required_actions)
+                    if not workflow.get_action(a).get('next', None):
+                        workflow.get_action(a)['next'] = set()
+                    workflow.get_action(a)['next'].add(action)
+            else:
+                workflow.root.add(action)
+
+        # The list of actions that needs to be preserved.
         workflow = deepcopy(self)
+        actions = set(map(lambda x: x[0], workflow.actions.items()))
 
-        # Make the list of the actions to be removed.
-        actions = list(map(lambda x: x[0], workflow.actions.items()))
-        actions.remove(action)
+        required_actions = set()
 
-        # Prepare the action for its execution only.
-        if workflow.get_action(action).get('next', None):
-            workflow.get_action(action)['next'] = set()
+        if with_dependencies:
+            # Prepare the graph for running only the given action
+            # only with its dependencies.
+            find_root_recursively(workflow, action, required_actions)
 
-        if workflow.get_action(action).get('needs', None):
-            workflow.get_action(action)['needs'] = list()
+            filtered_actions = actions - required_actions
+
+            for ra in required_actions:
+                a_block = workflow.get_action(ra)
+                common_actions = filtered_actions.intersection(
+                    a_block.get('next', set()))
+                if common_actions:
+                    for ca in common_actions:
+                        a_block['next'].remove(ca)
+        else:
+            # Prepare the action for its execution only.
+            required_actions.add(action)
 
-        workflow.root.add(action)
+            if workflow.get_action(action).get('next', None):
+                workflow.get_action(action)['next'] = set()
+
+            if workflow.get_action(action).get('needs', None):
+                workflow.get_action(action)['needs'] = list()
+
+            workflow.root.add(action)
+
+        # Make the list of the actions to be removed.
+        actions = actions - required_actions
 
         # Remove the remaining actions
         for a in actions: