Merge pull request #381 from JrGoodle/s3-coverage

Upload coverage results to s3

.coveragerc

@@ -5,3 +5,8 @@ parallel = True
 omit =
     # omit everything in /usr
     /usr/*
+
+[paths]
+source =
+    clowder/clowder
+    /home/travis/build/*/clowder/clowder

clowder/clowder/cli/branch_controller.py

@@ -12,7 +12,7 @@
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
     filter_groups,
-    filter_projects_on_project_names,
+    filter_projects,
     options_help_message,
     run_group_command,
     run_project_command
@@ -72,6 +72,6 @@ def _branch(self):
                 run_group_command(group, self.app.pargs.skip, 'branch', local=local, remote=remote)
             return
 
-        projects = filter_projects_on_project_names(CLOWDER_CONTROLLER.groups, self.app.pargs.projects)
+        projects = filter_projects(CLOWDER_CONTROLLER.groups, project_names=self.app.pargs.projects)
         for project in projects:
             run_project_command(project, self.app.pargs.skip, 'branch', local=local, remote=remote)

clowder/clowder/cli/checkout_controller.py

@@ -12,7 +12,7 @@
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
     filter_groups,
-    filter_projects_on_project_names,
+    filter_projects,
     options_help_message,
     run_group_command,
     run_project_command
@@ -65,6 +65,6 @@ def _checkout(self):
                 run_group_command(group, self.app.pargs.skip, 'checkout', self.app.pargs.branch[0])
             return
 
-        projects = filter_projects_on_project_names(CLOWDER_CONTROLLER.groups, self.app.pargs.projects)
+        projects = filter_projects(CLOWDER_CONTROLLER.groups, project_names=self.app.pargs.projects)
         for project in projects:
             run_project_command(project, self.app.pargs.skip, 'checkout', self.app.pargs.branch[0])

clowder/clowder/cli/clean_controller.py

@@ -12,7 +12,7 @@
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
     filter_groups,
-    filter_projects_on_project_names,
+    filter_projects,
     options_help_message,
     run_group_command,
     run_project_command
@@ -112,7 +112,7 @@ def _clean(clowder, group_names, **kwargs):
             run_group_command(group, skip, 'clean', args=args, recursive=recursive)
         return
 
-    projects = filter_projects_on_project_names(clowder.groups, project_names)
+    projects = filter_projects(clowder.groups, project_names=project_names)
     for project in projects:
         run_project_command(project, skip, 'clean', args=args, recursive=recursive)
 
@@ -139,6 +139,6 @@ def _clean_all(clowder, group_names, **kwargs):
             run_group_command(group, skip, 'clean_all')
         return
 
-    projects = filter_projects_on_project_names(clowder.groups, project_names)
+    projects = filter_projects(clowder.groups, project_names=project_names)
     for project in projects:
         run_project_command(project, skip, 'clean_all')

clowder/clowder/cli/diff_controller.py

@@ -12,7 +12,7 @@
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
     filter_groups,
-    filter_projects_on_project_names,
+    filter_projects,
     options_help_message,
     run_group_command,
     run_project_command
@@ -65,6 +65,6 @@ def _diff(self):
                 run_group_command(group, [], 'diff')
             return
 
-        projects = filter_projects_on_project_names(CLOWDER_CONTROLLER.groups, self.app.pargs.projects)
+        projects = filter_projects(CLOWDER_CONTROLLER.groups, project_names=self.app.pargs.projects)
         for project in projects:
             run_project_command(project, [], 'diff')

clowder/clowder/cli/parallel_commands.py

@@ -19,8 +19,7 @@
 from clowder.util.progress import Progress
 from clowder.util.clowder_utils import (
     filter_groups,
-    filter_projects_on_group_names,
-    filter_projects_on_project_names,
+    filter_projects,
     print_parallel_groups_output,
     print_parallel_projects_output,
     run_group_command,
@@ -122,7 +121,7 @@ def sig_int(signal_num, frame):
 def forall(clowder, command, ignore_errors, group_names, **kwargs):
     """Runs script in project directories specified
 
-    .. py:function:: forall_script(script_command, ignore_errors, group_names, project_names=None, skip=[], parallel=False)
+    .. py:function:: forall_script(clowder, command, ignore_errors, group_names, project_names=None, skip=[], parallel=False)
 
     :param ClowderController clowder: ClowderController instance
     :param list[str] command: Command or script and optional arguments
@@ -139,10 +138,7 @@ def forall(clowder, command, ignore_errors, group_names, **kwargs):
     skip = kwargs.get('skip', [])
     parallel = kwargs.get('parallel', False)
 
-    if project_names is None:
-        projects = filter_projects_on_group_names(clowder.groups, group_names)
-    else:
-        projects = filter_projects_on_project_names(clowder.groups, project_names)
+    projects = filter_projects(clowder.groups, group_names=group_names, project_names=project_names)
 
     if parallel:
         _forall_parallel([" ".join(command)], skip, ignore_errors, projects)
@@ -155,7 +151,7 @@ def forall(clowder, command, ignore_errors, group_names, **kwargs):
 def herd(clowder, group_names, **kwargs):
     """Clone projects or update latest from upstream
 
-    .. py:function:: herd(group_names, branch=None, tag=None, depth=0, rebase=False, project_names=None, skip=[], protocol=None)
+    .. py:function:: herd(clowder, group_names, branch=None, tag=None, depth=0, rebase=False, project_names=None, skip=[], protocol=None)
 
     :param ClowderController clowder: ClowderController instance
     :param list[str] group_names: Group names to herd
@@ -186,7 +182,7 @@ def herd(clowder, group_names, **kwargs):
                               depth=depth, rebase=rebase, protocol=protocol)
         return
 
-    projects = filter_projects_on_project_names(clowder.groups, project_names)
+    projects = filter_projects(clowder.groups, project_names=project_names)
     validate_projects(projects)
     for project in projects:
         run_project_command(project, skip, 'herd', branch=branch, tag=tag,
@@ -196,7 +192,7 @@ def herd(clowder, group_names, **kwargs):
 def herd_parallel(clowder, group_names, **kwargs):
     """Clone projects or update latest from upstream in parallel
 
-    .. py:function:: herd_parallel(group_names, branch=None, tag=None, depth=0, rebase=False, project_names=None, skip=[], protocol=None)
+    .. py:function:: herd_parallel(clowder, group_names, branch=None, tag=None, depth=0, rebase=False, project_names=None, skip=[], protocol=None)
 
     :param ClowderController clowder: ClowderController instance
     :param list[str] group_names: Group names to herd
@@ -220,15 +216,9 @@ def herd_parallel(clowder, group_names, **kwargs):
     protocol = kwargs.get('protocol', None)
 
     print(' - Herd projects in parallel\n')
-    if project_names is None:
-        groups = filter_groups(clowder.groups, group_names)
-        validate_groups(groups)
-        projects = filter_projects_on_group_names(clowder.groups, group_names)
-        print_parallel_groups_output(groups, skip)
-    else:
-        projects = filter_projects_on_project_names(clowder.groups, project_names)
-        validate_projects(projects)
-        print_parallel_projects_output(projects, skip)
+    _validate_print_output(clowder, group_names, project_names=project_names, skip=skip)
+
+    projects = filter_projects(clowder.groups, group_names=group_names, project_names=project_names)
 
     for project in projects:
         if project.name in skip:
@@ -243,7 +233,7 @@ def herd_parallel(clowder, group_names, **kwargs):
 def reset(clowder, group_names, **kwargs):
     """Reset project branches to upstream or checkout tag/sha as detached HEAD
 
-    .. py:function:: reset(group_names, timestamp_project=None, parallel=False, project_names=None, skip=[])
+    .. py:function:: reset(clowder, group_names, timestamp_project=None, parallel=False, project_names=None, skip=[])
 
     :param ClowderController clowder: ClowderController instance
     :param list[str] group_names: Group names to reset
@@ -274,7 +264,7 @@ def reset(clowder, group_names, **kwargs):
             run_group_command(group, skip, 'reset', timestamp=timestamp)
         return
 
-    projects = filter_projects_on_project_names(clowder.groups, project_names)
+    projects = filter_projects(clowder.groups, project_names=project_names)
     validate_projects(projects)
     for project in projects:
         run_project_command(project, skip, 'reset', timestamp=timestamp)
@@ -291,7 +281,7 @@ def sync(clowder, project_names, rebase=False, parallel=False):
     :param Optional[bool] parallel: Whether command is being run in parallel, affects output
     """
 
-    projects = filter_projects_on_project_names(clowder.groups, project_names)
+    projects = filter_projects(clowder.groups, project_names=project_names)
     if parallel:
         _sync_parallel(projects, rebase=rebase)
         return
@@ -333,7 +323,7 @@ def _forall_parallel(commands, skip, ignore_errors, projects):
 def _reset_parallel(clowder, group_names, **kwargs):
     """Reset project branches to upstream or checkout tag/sha as detached HEAD in parallel
 
-    .. py:function:: _reset_parallel(group_names, timestamp_project=None, project_names=None, skip=[])
+    .. py:function:: _reset_parallel(clowder, group_names, timestamp_project=None, project_names=None, skip=[])
 
     :param ClowderController clowder: ClowderController instance
     :param list[str] group_names: Group names to reset
@@ -353,15 +343,9 @@ def _reset_parallel(clowder, group_names, **kwargs):
     if timestamp_project:
         timestamp = clowder.get_timestamp(timestamp_project)
 
-    if project_names is None:
-        groups = filter_groups(clowder.groups, group_names)
-        validate_groups(groups)
-        projects = filter_projects_on_group_names(clowder.groups, group_names)
-        print_parallel_groups_output(groups, skip)
-    else:
-        projects = filter_projects_on_project_names(clowder.groups, project_names)
-        validate_projects(projects)
-        print_parallel_projects_output(projects, skip)
+    _validate_print_output(clowder, group_names, project_names=project_names, skip=skip)
+
+    projects = filter_projects(clowder.groups, group_names=group_names, project_names=project_names)
 
     for project in projects:
         if project.name in skip:
@@ -389,6 +373,34 @@ def _sync_parallel(projects, rebase=False):
     pool_handler(len(projects))
 
 
+def _validate_print_output(clowder, group_names, **kwargs):
+    """Validate projects/groups and print output
+
+    .. py:function:: _validate_print_output(clowder, group_names, project_names=None, skip=[])
+
+    :param ClowderController clowder: ClowderController instance
+    :param list[str] group_names: Group names to validate/print
+
+    Keyword Args:
+        project_names (list[str]): Project names to validate/print
+        skip (list[str]): Project names to skip
+    """
+
+    project_names = kwargs.get('project_names', None)
+    skip = kwargs.get('skip', [])
+
+    if project_names is None:
+        groups = filter_groups(clowder.groups, group_names)
+        validate_groups(groups)
+        projects = filter_projects(clowder.groups, group_names=group_names)
+        print_parallel_groups_output(groups, skip)
+        return
+
+    projects = filter_projects(clowder.groups, project_names=project_names)
+    validate_projects(projects)
+    print_parallel_projects_output(projects, skip)
+
+
 # Disable warnings shown by pylint for catching too general exception
 # pylint: disable=W0703
 

clowder/clowder/cli/prune_controller.py

@@ -19,7 +19,7 @@
     existing_branch_groups,
     existing_branch_projects,
     filter_groups,
-    filter_projects_on_project_names,
+    filter_projects,
     options_help_message,
     run_group_command,
     run_project_command,
@@ -129,7 +129,7 @@ def _prune(clowder, group_names, branch, **kwargs):
         _prune_groups(groups, branch, skip=skip, force=force, local=local, remote=remote)
         return
 
-    projects = filter_projects_on_project_names(clowder.groups, project_names)
+    projects = filter_projects(clowder.groups, project_names=project_names)
     validate_projects(projects)
     _prune_projects(projects, branch, skip=skip, force=force, local=local, remote=remote)
 

clowder/clowder/cli/start_controller.py

@@ -13,7 +13,7 @@
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
     filter_groups,
-    filter_projects_on_project_names,
+    filter_projects,
     options_help_message,
     run_group_command,
     run_project_command,
@@ -85,7 +85,7 @@ def _start_branches(self, tracking):
                 run_group_command(group, self.app.pargs.skip, 'start', self.app.pargs.branch, tracking)
             return
 
-        projects = filter_projects_on_project_names(CLOWDER_CONTROLLER.groups, self.app.pargs.projects)
+        projects = filter_projects(CLOWDER_CONTROLLER.groups, project_names=self.app.pargs.projects)
         validate_projects(projects)
         for project in projects:
             run_project_command(project, self.app.pargs.skip, 'start', self.app.pargs.branch, tracking)

clowder/clowder/cli/stash_controller.py

@@ -14,7 +14,7 @@
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
     filter_groups,
-    filter_projects_on_project_names,
+    filter_projects,
     options_help_message,
     run_group_command,
     run_project_command
@@ -70,6 +70,6 @@ def _stash(self):
                 run_group_command(group, self.app.pargs.skip, 'stash')
             return
 
-        projects = filter_projects_on_project_names(CLOWDER_CONTROLLER.groups, self.app.pargs.projects)
+        projects = filter_projects(CLOWDER_CONTROLLER.groups, project_names=self.app.pargs.projects)
         for project in projects:
             run_project_command(project, self.app.pargs.skip, 'stash')

clowder/clowder/util/clowder_utils.py

@@ -8,6 +8,7 @@
 import os
 
 import clowder.util.formatting as fmt
+from clowder import ROOT_DIR
 from clowder.error.clowder_exit import ClowderExit
 
 
@@ -37,6 +38,20 @@ def existing_branch_projects(projects, branch, is_remote):
     return any([p.existing_branch(branch, is_remote=is_remote) for p in projects])
 
 
+def get_clowder_yaml_import_path(import_name):
+    """Return path to imported clowder.yaml file
+
+    :param str import_name: Name of imported clowder.yaml
+    :return: Path to imported clowder.yaml file
+    :rtype: str
+    """
+
+    if import_name == 'default':
+        return os.path.join(ROOT_DIR, '.clowder', 'clowder.yaml')
+
+    return os.path.join(ROOT_DIR, '.clowder', 'versions', import_name, 'clowder.yaml')
+
+
 def filter_groups(groups, names):
     """Filter groups based on given group names
 
@@ -49,28 +64,23 @@ def filter_groups(groups, names):
     return [g for g in groups if g.name in names]
 
 
-def filter_projects_on_project_names(groups, names):
-    """Filter projects based on given project names
+def filter_projects(groups, group_names=None, project_names=None):
+    """Filter projects based on given project or group names
 
     :param list[Group] groups: Groups to filter
-    :param list[str] names: Project names to match against
+    :param list[str] group_names: Group names to match against
+    :param list[str] project_names: Project names to match against
     :return: List of projects in groups matching given names
     :rtype: list[Project]
     """
 
-    return [p for g in groups for p in g.projects if p.name in names]
+    if project_names is not None:
+        return [p for g in groups for p in g.projects if p.name in project_names]
 
+    if group_names is not None:
+        return [p for g in groups if g.name in group_names for p in g.projects]
 
-def filter_projects_on_group_names(groups, names):
-    """Filter projects based on given group names
-
-    :param list[Group] groups: Groups to filter
-    :param list[str] names: Group names to match against
-    :return: List of projects in groups matching given names
-    :rtype: list[Project]
-    """
-
-    return [p for g in groups if g.name in names for p in g.projects]
+    return []
 
 
 def get_saved_version_names():

clowder/clowder/yaml/loading.py

@@ -15,6 +15,7 @@
 from clowder import ROOT_DIR
 from clowder.error.clowder_exit import ClowderExit
 from clowder.error.clowder_yaml_error import ClowderYAMLError
+from clowder.util.clowder_utils import get_clowder_yaml_import_path
 from clowder.yaml import __MAX_IMPORT_DEPTH__
 from clowder.yaml.parsing import parse_yaml
 
@@ -35,13 +36,7 @@ def load_yaml():
             break
 
         imported_yaml_files.append(parsed_yaml)
-        imported_yaml = parsed_yaml['import']
-
-        if imported_yaml == 'default':
-            imported_yaml_file = os.path.join(ROOT_DIR, '.clowder', 'clowder.yaml')
-        else:
-            imported_yaml_file = os.path.join(ROOT_DIR, '.clowder', 'versions',
-                                              imported_yaml, 'clowder.yaml')
+        imported_yaml_file = get_clowder_yaml_import_path(parsed_yaml['import'])
 
         parsed_yaml = parse_yaml(imported_yaml_file)
         if len(imported_yaml_files) > __MAX_IMPORT_DEPTH__: