Merge pull request #541 from JrGoodle/jobs

Change --parallel to --jobs

clowder_test/clowder_test/execute.py

@@ -32,7 +32,7 @@ def execute_test_command(command: str, path: str, parallel: bool = False, write:
     test_env['ACCESS_LEVEL'] = 'write' if write else 'read'
 
     if parallel:
-        test_env['PARALLEL'] = '--parallel'
+        test_env['PARALLEL'] = '--jobs 4'
 
     if coverage:
         rc_file = os.path.join(ROOT_DIR, '.coveragerc')

src/clowder/__init__.py

@@ -61,8 +61,10 @@ def existing_clowder_repo(directory: Path) -> bool:
             print()
             exit(err.error_type.value)
     if clowder_yml.is_symlink():
+        # TODO: Check whether symlink source exists
         CLOWDER_YAML = clowder_yml
     elif clowder_yaml.is_symlink():
+        # TODO: Check whether symlink source exists
         CLOWDER_YAML = clowder_yaml
 
     clowder_versions = CLOWDER_REPO_DIR / 'versions'

src/clowder/cli/config.py

@@ -48,9 +48,9 @@ def add_config_clear_parser(subparsers: argparse._SubParsersAction) -> None: # n
     rebase_parser = config_clear_subparsers.add_parser('rebase', help='Clear use rebase with herd command')
     rebase_parser.set_defaults(func=config_clear_rebase)
 
-    # clowder config clear parallel
-    parallel_parser = config_clear_subparsers.add_parser('parallel', help='Clear use parallel commands')
-    parallel_parser.set_defaults(func=config_clear_parallel)
+    # clowder config clear jobs
+    jobs_parser = config_clear_subparsers.add_parser('jobs', help='Clear use jobs commands')
+    jobs_parser.set_defaults(func=config_clear_jobs)
 
     # clowder config clear projects
     projects_parser = config_clear_subparsers.add_parser('projects', help='Clear default projects and groups')
@@ -77,9 +77,9 @@ def add_config_get_parser(subparsers: argparse._SubParsersAction) -> None: # noq
     rebase_parser = config_get_subparsers.add_parser('rebase', help='Get use rebase with herd command')
     rebase_parser.set_defaults(func=config_get_rebase)
 
-    # clowder config get parallel
-    parallel_parser = config_get_subparsers.add_parser('parallel', help='Get use parallel commands')
-    parallel_parser.set_defaults(func=config_get_parallel)
+    # clowder config get jobs
+    jobs_parser = config_get_subparsers.add_parser('jobs', help='Get use jobs commands')
+    jobs_parser.set_defaults(func=config_get_jobs)
 
     # clowder config get projects
     projects_parser = config_get_subparsers.add_parser('projects', help='Get default projects and groups')
@@ -104,9 +104,9 @@ def add_config_set_parser(subparsers: argparse._SubParsersAction) -> None: # noq
     rebase_parser = config_set_subparsers.add_parser('rebase', help='Set use rebase with herd command')
     rebase_parser.set_defaults(func=config_set_rebase)
 
-    # clowder config set parallel
-    parallel_parser = config_set_subparsers.add_parser('parallel', help='Set use parallel commands')
-    parallel_parser.set_defaults(func=config_set_parallel)
+    # clowder config set jobs
+    jobs_parser = config_set_subparsers.add_parser('jobs', help='Set use jobs commands')
+    jobs_parser.set_defaults(func=config_set_jobs)
 
     # clowder config set projects
     projects_arguments = [
@@ -142,12 +142,12 @@ def config_clear_all(args) -> None: # noqa
 
 @valid_clowder_yaml_required
 @print_clowder_name
-def config_clear_parallel(args) -> None: # noqa
-    """Clowder config clear parallel command entry point"""
+def config_clear_jobs(args) -> None: # noqa
+    """Clowder config clear jobs command entry point"""
 
-    print(' - Clear parallel config value')
+    print(' - Clear jobs config value')
     config = Config(CLOWDER_CONTROLLER.name, CLOWDER_CONTROLLER.project_choices)
-    config.current_clowder_config.parallel = None
+    config.current_clowder_config.jobs = None
     config.save()
     print()
     config.current_clowder_config.print_configuration()
@@ -205,12 +205,12 @@ def config_get_all(args) -> None: # noqa
 
 @valid_clowder_yaml_required
 @print_clowder_name
-def config_get_parallel(args) -> None: # noqa
-    """Clowder config get parallel command entry point"""
+def config_get_jobs(args) -> None: # noqa
+    """Clowder config get jobs command entry point"""
 
-    print(' - Get parallel config value')
+    print(' - Get jobs config value')
     config = Config(CLOWDER_CONTROLLER.name, CLOWDER_CONTROLLER.project_choices)
-    config.current_clowder_config.print_config_value(ClowderConfigType.PARALLEL)
+    config.current_clowder_config.print_config_value(ClowderConfigType.JOBS)
 
 
 @valid_clowder_yaml_required
@@ -245,12 +245,12 @@ def config_get_rebase(args) -> None: # noqa
 
 @valid_clowder_yaml_required
 @print_clowder_name
-def config_set_parallel(args) -> None: # noqa
-    """Clowder config set parallel command entry point"""
+def config_set_jobs(args) -> None: # noqa
+    """Clowder config set jobs command entry point"""
 
-    print(' - Set parallel config value')
+    print(' - Set jobs config value')
     config = Config(CLOWDER_CONTROLLER.name, CLOWDER_CONTROLLER.project_choices)
-    config.current_clowder_config.parallel = True
+    config.current_clowder_config.jobs = True
     config.save()
     print()
     config.current_clowder_config.print_configuration()

src/clowder/cli/forall.py

@@ -7,7 +7,7 @@
 
 import argparse
 import os
-from typing import List
+from typing import List, Optional
 
 import clowder.util.formatting as fmt
 from clowder.clowder_controller import CLOWDER_CONTROLLER
@@ -37,7 +37,8 @@ def add_forall_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
         (['--command', '-c'], dict(nargs='+', metavar='COMMAND', default=None,
                                    help='command or script to run in project directories')),
         (['--ignore-errors', '-i'], dict(action='store_true', help='ignore errors in command or script')),
-        (['--parallel', '-p'], dict(action='store_true', help='run commands in parallel'))
+        (['--jobs', '-j'], dict(metavar='JOBS', nargs=1, default=None, type=int,
+                                help='number of jobs to use runnning commands in parallel')),
     ]
 
     parser = subparsers.add_parser('forall', help='Run command or script in project directories')
@@ -51,29 +52,35 @@ def add_forall_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
 def forall(args) -> None:
     """Clowder forall command private implementation"""
 
-    _forall_impl(args.command, args.ignore_errors, projects=args.projects, parallel=args.parallel)
+    jobs = None
+    if args.jobs:
+        jobs = args.jobs[0]
 
+    _forall_impl(args.command, args.ignore_errors, projects=args.projects, jobs=jobs)
 
-def _forall_impl(command: List[str], ignore_errors: bool, projects: List[str], parallel: bool = False) -> None:
+
+def _forall_impl(command: List[str], ignore_errors: bool, projects: List[str], jobs: Optional[int] = None) -> None:
     """Runs script in project directories specified
 
     :param list[str] command: Command or script and optional arguments
     :param bool ignore_errors: Whether to exit if command returns a non-zero exit code
     :param List[str] projects: Project names to clean
-    :param bool parallel: Whether command is being run in parallel, affects output
+    :param Optional[int] jobs: Number of jobs to use running parallel commands
     """
 
     config = Config(CLOWDER_CONTROLLER.name, CLOWDER_CONTROLLER.project_choices)
     projects = config.process_projects_arg(projects)
     projects = filter_projects(CLOWDER_CONTROLLER.projects, projects)
 
-    parallel_config = config.current_clowder_config.parallel
-    parallel = parallel_config if parallel_config is not None else parallel
+    jobs_config = config.current_clowder_config.jobs
+    jobs = jobs_config if jobs_config is not None else jobs
 
-    if parallel and os.name == "posix":
-        forall_parallel([" ".join(command)], projects, ignore_errors)
+    if jobs is not None and jobs != 1 and os.name == "posix":
+        if jobs <= 0:
+            jobs = 4
+        forall_parallel([" ".join(command)], projects, jobs, ignore_errors=ignore_errors)
         return
 
     for project in projects:
         print(project.status())
-        project.run([" ".join(command)], ignore_errors)
+        project.run([" ".join(command)], ignore_errors=ignore_errors)

src/clowder/cli/herd.py

@@ -37,7 +37,8 @@ def add_herd_parser(subparsers: argparse._SubParsersAction) -> None: # noqa
                             choices=CLOWDER_CONTROLLER.project_choices_with_default,
                             help=fmt.options_help_message(CLOWDER_CONTROLLER.project_choices,
                                                           'projects and groups to show branches for'))),
-        (['--parallel', '-p'], dict(action='store_true', help='run commands in parallel')),
+        (['--jobs', '-j'], dict(metavar='JOBS', nargs=1, default=None, type=int,
+                                help='number of jobs to use runnning commands in parallel')),
         (['--rebase', '-r'], dict(action='store_true', help='use rebase instead of pull')),
         (['--depth', '-d'], dict(default=None, type=int, nargs=1, metavar='DEPTH', help='depth to herd'))
     ]
@@ -72,14 +73,20 @@ def herd(args) -> None:
     rebase_config = config.current_clowder_config.rebase
     rebase = rebase_config if rebase_config is not None else rebase
 
-    parallel_config = config.current_clowder_config.parallel
-    parallel = parallel_config if parallel_config is not None else args.parallel
+    jobs = None
+    if args.jobs:
+        jobs = args.jobs[0]
+
+    jobs_config = config.current_clowder_config.jobs
+    jobs = jobs_config if jobs_config is not None else jobs
 
     projects = config.process_projects_arg(args.projects)
     projects = filter_projects(CLOWDER_CONTROLLER.projects, projects)
 
-    if parallel and os.name == "posix":
-        herd_parallel(projects, branch=branch, tag=tag, depth=depth, rebase=rebase)
+    if jobs is not None and jobs != 1 and os.name == "posix":
+        if jobs <= 0:
+            jobs = 4
+        herd_parallel(projects, jobs, branch=branch, tag=tag, depth=depth, rebase=rebase)
         return
 
     validate_project_statuses(projects)

src/clowder/cli/reset.py

@@ -38,7 +38,8 @@ def add_reset_parser(subparsers: argparse._SubParsersAction): # noqa
                             choices=CLOWDER_CONTROLLER.project_choices_with_default,
                             help=fmt.options_help_message(CLOWDER_CONTROLLER.project_choices,
                                                           'projects and groups to reset'))),
-        (['--parallel', '-p'], dict(action='store_true', help='run commands in parallel')),
+        (['--jobs', '-j'], dict(metavar='JOBS', nargs=1, default=None, type=int,
+                                help='number of jobs to use runnning commands in parallel')),
         (['--timestamp', '-t'], dict(choices=CLOWDER_CONTROLLER.project_names,
                                      default=None, nargs=1, metavar='TIMESTAMP',
                                      help='project to reset timestamps relative to'))
@@ -60,27 +61,34 @@ def reset(args) -> None:
     timestamp_project = None
     if args.timestamp:
         timestamp_project = args.timestamp[0]
-    _reset_impl(args.projects, timestamp_project=timestamp_project, parallel=args.parallel)
 
+    jobs = None
+    if args.jobs:
+        jobs = args.jobs[0]
 
-def _reset_impl(project_names: List[str], timestamp_project: Optional[str] = None, parallel: bool = False) -> None:
+    _reset_impl(args.projects, timestamp_project=timestamp_project, jobs=jobs)
+
+
+def _reset_impl(project_names: List[str], timestamp_project: Optional[str] = None, jobs: Optional[int] = None) -> None:
     """Reset project branches to upstream or checkout tag/sha as detached HEAD
 
     :param List[str] project_names: Project names to reset
     :param Optional[str] timestamp_project: Reference project to checkout other project commit timestamps relative to
-    :param bool parallel: Whether command is being run in parallel, affects output
+    :param Optional[int] jobs: Number of jobs to use runnning commands in parallel
     """
 
     config = Config(CLOWDER_CONTROLLER.name, CLOWDER_CONTROLLER.project_choices)
 
-    parallel_config = config.current_clowder_config.parallel
-    parallel = parallel_config if parallel_config is not None else parallel
+    jobs_config = config.current_clowder_config.jobs
+    jobs = jobs_config if jobs_config is not None else jobs
 
     projects = config.process_projects_arg(project_names)
     projects = filter_projects(CLOWDER_CONTROLLER.projects, projects)
 
-    if parallel and os.name == "posix":
-        reset_parallel(projects, timestamp_project=timestamp_project)
+    if jobs is not None and jobs != 1 and os.name == "posix":
+        if jobs <= 0:
+            jobs = 4
+        reset_parallel(projects, jobs, timestamp_project=timestamp_project)
         return
 
     timestamp = None

src/clowder/config/clowder_config.py

@@ -18,7 +18,7 @@
 
 @unique
 class ClowderConfigType(Enum):
-    PARALLEL = auto()
+    JOBS = auto()
     PROJECTS = auto()
     PROTOCOL = auto()
     REBASE = auto()
@@ -32,7 +32,7 @@ class ClowderConfig(object):
     :ivar Optional[Tuple[str, ...]] projects: Default projects
     :ivar Optional[str] protocol: Default protocol
     :ivar Optional[bool] rebase: Default rebase
-    :ivar Optional[bool] parallel: Default parallel
+    :ivar Optional[int] jobs: Default number of jobs
     """
 
     def __init__(self, clowder_config: Optional[dict] = None,
@@ -47,7 +47,7 @@ def __init__(self, clowder_config: Optional[dict] = None,
         if clowder_config is None:
             self.clowder_dir = CLOWDER_DIR
             self.name = current_clowder_name
-            self.parallel = None
+            self.jobs = None
             self.projects = None
             self.protocol = None
             self.rebase = None
@@ -67,7 +67,7 @@ def __init__(self, clowder_config: Optional[dict] = None,
             self.projects: Optional[Tuple[str, ...]] = None if projects is None else tuple(sorted(projects))
             self.protocol: Optional[str] = defaults.get('protocol', None)
             self.rebase: Optional[bool] = defaults.get('rebase', None)
-            self.parallel: Optional[bool] = defaults.get('parallel', None)
+            self.jobs: Optional[int] = defaults.get('jobs', None)
 
     def is_empty(self) -> bool:
         """Determine if any config values are set
@@ -76,7 +76,7 @@ def is_empty(self) -> bool:
         :rtype: bool
         """
 
-        config_status = [self.is_config_value_set(ClowderConfigType.PARALLEL),
+        config_status = [self.is_config_value_set(ClowderConfigType.JOBS),
                          self.is_config_value_set(ClowderConfigType.PROJECTS),
                          self.is_config_value_set(ClowderConfigType.PROTOCOL),
                          self.is_config_value_set(ClowderConfigType.REBASE)]
@@ -85,7 +85,7 @@ def is_empty(self) -> bool:
     def clear(self) -> None:
         """Clear all config settings"""
 
-        self.parallel = None
+        self.jobs = None
         self.projects = None
         self.protocol = None
         self.rebase = None
@@ -104,8 +104,8 @@ def get_yaml(self) -> dict:
             defaults['protocol'] = self.protocol
         if self.rebase is not None:
             defaults['rebase'] = self.rebase
-        if self.parallel is not None:
-            defaults['parallel'] = self.parallel
+        if self.jobs is not None:
+            defaults['jobs'] = self.jobs
         config = {'clowder_dir': str(self.clowder_dir),
                   'name': self.name,
                   'defaults': defaults}
@@ -117,8 +117,8 @@ def is_config_value_set(self, value: ClowderConfigType) -> None:
         :raise ClowderError:
         """
 
-        if value is ClowderConfigType.PARALLEL:
-            return self.parallel is not None
+        if value is ClowderConfigType.JOBS:
+            return self.jobs is not None
         elif value is ClowderConfigType.PROJECTS:
             return self.projects is not None
         elif value is ClowderConfigType.PROTOCOL:
@@ -134,11 +134,11 @@ def print_config_value(self, value: ClowderConfigType) -> None:
         :raise ClowderError:
         """
 
-        if value is ClowderConfigType.PARALLEL:
-            if self.parallel is None:
-                print(" - parallel not set")
+        if value is ClowderConfigType.JOBS:
+            if self.jobs is None:
+                print(" - jobs not set")
             else:
-                print(f" - parallel: {self.parallel}")
+                print(f" - jobs: {self.jobs}")
         elif value is ClowderConfigType.PROJECTS:
             if self.projects is None:
                 print(" - projects not set")
@@ -167,8 +167,8 @@ def print_configuration(self) -> None:
             return
 
         output = ''
-        if self.parallel is not None:
-            output += f" - parallel: {self.parallel}\n"
+        if self.jobs is not None:
+            output += f" - jobs: {self.jobs}\n"
         if self.projects is not None:
             output += f" - projects: {', '.join(self.projects)}\n"
         if self.protocol is not None:

src/clowder/util/clowder.config.schema.json

@@ -18,6 +18,11 @@
       "minLength": 1,
       "pattern": "^\/.+"
     },
+    "jobs": {
+      "description": "Number of jobs to use for parallel commands",
+      "type": "integer",
+      "minimum": 1
+    },
     "stringArrayUnique": {
       "type": "array",
       "uniqueItems": true,
@@ -64,10 +69,8 @@
           "type": "boolean",
           "default": false
         },
-        "parallel": {
-          "description": "Whether to run parallel commands by default",
-          "type": "boolean",
-          "default": false
+        "jobs": {
+          "$ref": "#/definitions/jobs"
         }
       }
     },