Merge pull request #375 from JrGoodle/forall-params

Add the ability to pass arguments to forall scripts

clowder/clowder/cli/forall_controller.py

@@ -28,7 +28,7 @@ class Meta:
     @expose(
         help='Run command or script in project directories',
         arguments=[
-            (['--command', '-c'], dict(nargs=1, metavar='COMMAND',
+            (['--command', '-c'], dict(nargs='+', metavar='COMMAND', default=None,
                                        help='command or script to run in project directories')),
             (['--ignore-errors', '-i'], dict(action='store_true', help='ignore errors in command or script')),
             (['--parallel'], dict(action='store_true', help='run commands in parallel')),
@@ -57,6 +57,6 @@ def forall(self):
     def _forall(self):
         """Clowder forall command private implementation"""
 
-        forall(CLOWDER_CONTROLLER, self.app.pargs.command[0], self.app.pargs.ignore_errors,
+        forall(CLOWDER_CONTROLLER, self.app.pargs.command, self.app.pargs.ignore_errors,
                group_names=self.app.pargs.groups, project_names=self.app.pargs.projects,
                skip=self.app.pargs.skip, parallel=self.app.pargs.parallel)

clowder/clowder/cli/parallel.py

@@ -54,15 +54,15 @@ def reset_project(project, timestamp):
     project.reset(timestamp=timestamp, parallel=True)
 
 
-def run_project(project, command, ignore_errors):
+def run_project(project, commands, ignore_errors):
     """Run command wrapper function for multiprocessing Pool execution
 
     :param Project project: Project instance
-    :param str command: Command to run
+    :param list[str] commands: Commands to run
     :param bool ignore_errors: Whether to exit if command returns a non-zero exit code
     """
 
-    project.run(command, ignore_errors, parallel=True)
+    project.run(commands, ignore_errors, parallel=True)
 
 
 def sync_project(project, rebase):
@@ -120,12 +120,12 @@ def sig_int(signal_num, frame):
 
 
 def forall(clowder, command, ignore_errors, group_names, **kwargs):
-    """Runs command or script in project directories specified
+    """Runs script in project directories specified
 
-    .. py:function:: forall(command, ignore_errors, group_names, project_names=None, skip=[], parallel=False)
+    .. py:function:: forall_script(script_command, ignore_errors, group_names, project_names=None, skip=[], parallel=False)
 
     :param ClowderController clowder: ClowderController instance
-    :param str command: Command to run
+    :param list[str] command: Command or script and optional arguments
     :param bool ignore_errors: Whether to exit if command returns a non-zero exit code
     :param list[str] group_names: Group names to run command for
 
@@ -145,11 +145,11 @@ def forall(clowder, command, ignore_errors, group_names, **kwargs):
         projects = filter_projects_on_project_names(clowder.groups, project_names)
 
     if parallel:
-        _forall_parallel(command, skip, ignore_errors, projects)
+        _forall_parallel([" ".join(command)], skip, ignore_errors, projects)
         return
 
     for project in projects:
-        run_project_command(project, skip, 'run', command, ignore_errors)
+        run_project_command(project, skip, 'run', [" ".join(command)], ignore_errors)
 
 
 def herd(clowder, group_names, **kwargs):
@@ -306,10 +306,10 @@ def sync(clowder, project_names, rebase=False, parallel=False):
         project.sync(rebase=rebase)
 
 
-def _forall_parallel(command, skip, ignore_errors, projects):
+def _forall_parallel(commands, skip, ignore_errors, projects):
     """Runs command or script for projects in parallel
 
-    :param str command: Command to run
+    :param list[str] commands: Command to run
     :param list[str] skip: Project names to skip
     :param bool ignore_errors: Whether to exit if command returns a non-zero exit code
     :param list[Project] projects: Projects to run command for
@@ -323,11 +323,12 @@ def _forall_parallel(command, skip, ignore_errors, projects):
         if not os.path.isdir(project.full_path()):
             cprint(" - Project is missing", 'red')
 
-    print('\n' + fmt.command(command))
+    for cmd in commands:
+        print('\n' + fmt.command(cmd))
     for project in projects:
         if project.name in skip:
             continue
-        result = __clowder_pool__.apply_async(run_project, args=(project, command, ignore_errors),
+        result = __clowder_pool__.apply_async(run_project, args=(project, commands, ignore_errors),
                                               callback=async_callback)
         __clowder_results__.append(result)
 

clowder/clowder/model/project.py

@@ -345,12 +345,12 @@ def reset(self, timestamp=None, parallel=False):
         repo = self._repo(self.full_path(), self._remote, self._ref, self._recursive, parallel=parallel)
         self._reset(repo, timestamp=timestamp)
 
-    def run(self, command, ignore_errors, parallel=False):
+    def run(self, commands, ignore_errors, parallel=False):
         """Run command or script in project directory
 
-        .. py:function:: run(command, ignore_errors, parallel=False)
+        .. py:function:: run(commands, ignore_errors, parallel=False)
 
-        :param str command: Command to run
+        :param list[str] commands: Command to run
         :param bool ignore_errors: Whether to exit if command returns a non-zero exit code
         :param Optional[bool] parallel: Whether command is being run in parallel, affects output
         """
@@ -361,7 +361,6 @@ def run(self, command, ignore_errors, parallel=False):
                 return
 
         self._print_output = not parallel
-        self._print(fmt.command(command))
 
         forall_env = {'CLOWDER_PATH': self._root_directory,
                       'PROJECT_PATH': self.full_path(),
@@ -372,13 +371,15 @@ def run(self, command, ignore_errors, parallel=False):
         if self.fork:
             forall_env['FORK_REMOTE'] = self.fork.remote_name
 
-        try:
-            execute_forall_command(command.split(), self.full_path(), forall_env, self._print_output)
-        except ClowderError:
-            if not ignore_errors:
-                err = fmt.command_failed_error(command)
-                self._print(err)
-                self._exit(err, parallel=parallel)
+        for cmd in commands:
+            self._print(fmt.command(cmd))
+            try:
+                execute_forall_command(cmd, self.full_path(), forall_env, self._print_output)
+            except ClowderError:
+                if not ignore_errors:
+                    err = fmt.command_failed_error(cmd)
+                    self._print(err)
+                    self._exit(err, parallel=parallel)
 
     @project_repo_exists
     def start(self, branch, tracking):

test/scripts/cats/forall.sh

@@ -26,13 +26,16 @@ test_forall_branches() {
         test_branch v0.1
         popd || exit 1
     done
+    clowder herd $PARALLEL || exit 1
 }
 test_forall_branches
 
 test_forall_command() {
     print_single_separator
     echo "TEST: Run forall command"
     clowder forall $PARALLEL -c 'git status' || exit 1
+    # echo "TEST: Run forall command with multiple arguments"
+    # clowder forall $PARALLEL -c 'git status' 'echo "hi"'|| exit 1
     echo "TEST: Run forall command for specific groups"
     clowder forall $PARALLEL -c 'git status' -g "$@" || exit 1
     echo "TEST: Run forall command with error"
@@ -46,6 +49,12 @@ test_forall_script() {
     print_single_separator
     echo "TEST: Run forall script"
     clowder forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script.sh" || exit 1
+    echo "TEST: Run forall script with arguments"
+    clowder forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script_args.sh" "one" "two" || exit 1
+    echo "TEST: Fail running forall script with arguments"
+    clowder forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script_args.sh" "one" && exit 1
+    echo "TEST: Ignore failures running forall script with arguments"
+    clowder forall $PARALLEL -ic "$TEST_SCRIPT_DIR/test_forall_script_args.sh" "one" || exit 1
     echo "TEST: Run forall script for specific groups"
     clowder forall $PARALLEL -c "$TEST_SCRIPT_DIR/test_forall_script.sh" -g "$@" || exit 1
     echo "TEST: Run forall script with error"

test/scripts/test_forall_script_args.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ]; then
+    exit 1
+fi
+
+if [ "$1" != 'one' ]; then
+    exit 1
+fi
+
+if [ -z "$2" ]; then
+    exit 1
+fi
+
+if [ "$2" != 'two' ]; then
+    exit 1
+fi