Merge pull request #460 from JrGoodle/new-schema

Update for new schema

clowder_test/clowder_test/cli/cats_controller.py

@@ -178,15 +178,6 @@ def save(self) -> None:
 
         self._execute_command('./save.sh', self.path)
 
-    @expose(
-        help='Run cats skip tests'
-    )
-    @create_cats_cache
-    def skip(self) -> None:
-        """clowder cats skip tests"""
-
-        self._execute_command('./skip.sh', self.path)
-
     @expose(
         help='Run cats start tests'
     )

script/clean

@@ -46,6 +46,7 @@ clean_examples() {
     "$examples_path/cats/clean.sh"
     "$examples_path/llvm-projects/clean.sh"
     "$examples_path/swift-projects/clean.sh"
+    "$examples_path/misc/clean.sh"
     if [ -z "$TRAVIS_OS_NAME" ] &&  [ -z "$CIRCLECI" ]; then
         echo 'Clean home directory examples...'
         rm -rf "$HOME/.clowder_tests"

src/clowder/cli/branch_controller.py

@@ -11,11 +11,8 @@
 from clowder.clowder_repo import print_clowder_repo_status
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
-    filter_groups,
     filter_projects,
-    options_help_message,
-    run_group_command,
-    run_project_command
+    options_help_message
 )
 
 
@@ -35,19 +32,10 @@ class Meta:
         arguments=[
             (['--all', '-a'], dict(action='store_true', help='show local and remote branches')),
             (['--remote', '-r'], dict(action='store_true', help='show remote branches')),
-            (['--groups', '-g'], dict(choices=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      default=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      nargs='+', metavar='GROUP',
-                                      help=options_help_message(CLOWDER_CONTROLLER.get_all_group_names(),
-                                                                'groups to show branches for'))),
             (['--projects', '-p'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                        nargs='+', metavar='PROJECT',
+                                        default=['all'], nargs='+', metavar='PROJECT',
                                         help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                                  'projects to show branches for'))),
-            (['--skip', '-s'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                    nargs='+', metavar='PROJECT', default=[],
-                                    help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                              'projects to skip')))
+                                                                  'projects to show branches for')))
             ]
     )
     def branch(self) -> None:
@@ -66,12 +54,7 @@ def _branch(self) -> None:
         elif self.app.pargs.remote:
             remote = True
 
-        if self.app.pargs.projects is None:
-            groups = filter_groups(CLOWDER_CONTROLLER.groups, self.app.pargs.groups)
-            for group in groups:
-                run_group_command(group, self.app.pargs.skip, 'branch', local=local, remote=remote)
-            return
-
-        projects = filter_projects(CLOWDER_CONTROLLER.groups, project_names=self.app.pargs.projects)
+        projects = filter_projects(CLOWDER_CONTROLLER.projects, self.app.pargs.projects)
         for project in projects:
-            run_project_command(project, self.app.pargs.skip, 'branch', local=local, remote=remote)
+            print(project.status())
+            project.branch(local=local, remote=remote)

src/clowder/cli/checkout_controller.py

@@ -11,11 +11,8 @@
 from clowder.clowder_repo import print_clowder_repo_status
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
-    filter_groups,
     filter_projects,
-    options_help_message,
-    run_group_command,
-    run_project_command
+    options_help_message
 )
 
 
@@ -34,19 +31,10 @@ class Meta:
         help='Checkout local branch in projects',
         arguments=[
             (['branch'], dict(nargs=1, action='store', help='branch to checkout', metavar='BRANCH')),
-            (['--groups', '-g'], dict(choices=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      default=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      nargs='+', metavar='GROUP',
-                                      help=options_help_message(CLOWDER_CONTROLLER.get_all_group_names(),
-                                                                'groups to checkout branches for'))),
             (['--projects', '-p'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                        nargs='+', metavar='PROJECT',
+                                        default=['all'], nargs='+', metavar='PROJECT',
                                         help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                                  'projects to checkout branches for'))),
-            (['--skip', '-s'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                    nargs='+', metavar='PROJECT', default=[],
-                                    help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                              'projects to skip')))
+                                                                  'projects to checkout branches for')))
         ]
     )
     def checkout(self) -> None:
@@ -59,12 +47,7 @@ def checkout(self) -> None:
     def _checkout(self) -> None:
         """Clowder checkout command private implementation"""
 
-        if self.app.pargs.projects is None:
-            groups = filter_groups(CLOWDER_CONTROLLER.groups, self.app.pargs.groups)
-            for group in groups:
-                run_group_command(group, self.app.pargs.skip, 'checkout', self.app.pargs.branch[0])
-            return
-
-        projects = filter_projects(CLOWDER_CONTROLLER.groups, project_names=self.app.pargs.projects)
+        projects = filter_projects(CLOWDER_CONTROLLER.projects, self.app.pargs.projects)
         for project in projects:
-            run_project_command(project, self.app.pargs.skip, 'checkout', self.app.pargs.branch[0])
+            print(project.status())
+            project.checkout(self.app.pargs.branch[0])

src/clowder/cli/clean_controller.py

@@ -5,19 +5,16 @@
 
 """
 
-from typing import List, Optional
+from typing import List
 
 from cement.ext.ext_argparse import ArgparseController, expose
 
 from clowder.clowder_controller import CLOWDER_CONTROLLER, ClowderController
 from clowder.clowder_repo import print_clowder_repo_status
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
-    filter_groups,
     filter_projects,
-    options_help_message,
-    run_group_command,
-    run_project_command
+    options_help_message
 )
 
 
@@ -41,19 +38,10 @@ class Meta:
             (['-f'], dict(action='store_true', help='remove directories with .git subdirectory or file')),
             (['-X'], dict(action='store_true', help='remove only files ignored by git')),
             (['-x'], dict(action='store_true', help='remove all untracked files')),
-            (['--groups', '-g'], dict(choices=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      default=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      nargs='+', metavar='GROUP',
-                                      help=options_help_message(CLOWDER_CONTROLLER.get_all_group_names(),
-                                                                'groups to clean'))),
             (['--projects', '-p'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                        nargs='+', metavar='PROJECT',
+                                        default=['all'], nargs='+', metavar='PROJECT',
                                         help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                                  'projects to clean'))),
-            (['--skip', '-s'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                    nargs='+', metavar='PROJECT', default=[],
-                                    help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                              'projects to skip')))
+                                                                  'projects to clean')))
             ]
     )
     def clean(self) -> None:
@@ -67,8 +55,10 @@ def _clean(self) -> None:
         """Clowder clean command private implementation"""
 
         if self.app.pargs.all:
-            _clean_all(CLOWDER_CONTROLLER, group_names=self.app.pargs.groups,
-                       project_names=self.app.pargs.projects, skip=self.app.pargs.skip)
+            projects = filter_projects(CLOWDER_CONTROLLER.projects, self.app.pargs.projects)
+            for project in projects:
+                print(project.status())
+                project.clean_all()
             return
 
         clean_args = ''
@@ -80,57 +70,23 @@ def _clean(self) -> None:
             clean_args += 'X'
         if self.app.pargs.x:
             clean_args += 'x'
-        _clean(CLOWDER_CONTROLLER, group_names=self.app.pargs.groups, project_names=self.app.pargs.projects,
-               skip=self.app.pargs.skip, clean_args=clean_args, recursive=self.app.pargs.recursive)
+        _clean(CLOWDER_CONTROLLER, self.app.pargs.projects, clean_args=clean_args, recursive=self.app.pargs.recursive)
 
 
-def _clean(clowder: ClowderController, group_names: List[str], clean_args: str = '', recursive: bool = False,
-           project_names: Optional[List[str]] = None, skip: Optional[List[str]] = None) -> None:
+def _clean(clowder: ClowderController, project_names: List[str], clean_args: str = '', recursive: bool = False) -> None:
     """Discard changes
 
     :param ClowderController clowder: ClowderController instance
-    :param list[str] group_names: Group names to clean
+    :param List[str] project_names: Project names to clean
     :param str clean_args: Git clean options
         - ``d`` Remove untracked directories in addition to untracked files
         - ``f`` Delete directories with .git sub directory or file
         - ``X`` Remove only files ignored by git
         - ``x`` Remove all untracked files
     :param bool recursive: Clean submodules recursively
-    :param Optional[List[str]] project_names: Project names to clean
-    :param Optional[List[str]] skip: Project names to skip
     """
 
-    skip = [] if skip is None else skip
-
-    if project_names is None:
-        groups = filter_groups(clowder.groups, group_names)
-        for group in groups:
-            run_group_command(group, skip, 'clean', args=clean_args, recursive=recursive)
-        return
-
-    projects = filter_projects(clowder.groups, project_names=project_names)
-    for project in projects:
-        run_project_command(project, skip, 'clean', args=clean_args, recursive=recursive)
-
-
-def _clean_all(clowder: ClowderController, group_names: List[str],
-               project_names: Optional[List[str]] = None, skip: Optional[List[str]] = None) -> None:
-    """Discard all changes
-
-    :param ClowderController clowder: ClowderController instance
-    :param list[str] group_names: Group names to clean
-    :param Optional[List[str]] project_names: Project names to clean
-    :param Optional[List[str]] skip: Project names to skip
-    """
-
-    skip = [] if skip is None else skip
-
-    if project_names is None:
-        groups = filter_groups(clowder.groups, group_names)
-        for group in groups:
-            run_group_command(group, skip, 'clean_all')
-        return
-
-    projects = filter_projects(clowder.groups, project_names=project_names)
+    projects = filter_projects(clowder.projects, project_names)
     for project in projects:
-        run_project_command(project, skip, 'clean_all')
+        print(project.status())
+        project.clean(args=clean_args, recursive=recursive)

src/clowder/cli/diff_controller.py

@@ -11,11 +11,8 @@
 from clowder.clowder_repo import print_clowder_repo_status
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.clowder_utils import (
-    filter_groups,
     filter_projects,
-    options_help_message,
-    run_group_command,
-    run_project_command
+    options_help_message
 )
 
 
@@ -33,19 +30,10 @@ class Meta:
     @expose(
         help='Show git diff for projects',
         arguments=[
-            (['--groups', '-g'], dict(choices=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      default=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      nargs='+', metavar='GROUP',
-                                      help=options_help_message(CLOWDER_CONTROLLER.get_all_group_names(),
-                                                                'groups to show diff for'))),
             (['--projects', '-p'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                        nargs='+', metavar='PROJECT',
+                                        default=['all'], nargs='+', metavar='PROJECT',
                                         help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                                  'projects to show diff for'))),
-            (['--skip', '-s'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                    nargs='+', metavar='PROJECT', default=[],
-                                    help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                              'projects to skip')))
+                                                                  'projects to show diff for')))
         ]
     )
     def diff(self) -> None:
@@ -58,12 +46,7 @@ def diff(self) -> None:
     def _diff(self) -> None:
         """Clowder diff command private implementation"""
 
-        if self.app.pargs.projects is None:
-            groups = filter_groups(CLOWDER_CONTROLLER.groups, self.app.pargs.groups)
-            for group in groups:
-                run_group_command(group, self.app.pargs.skip, 'diff')
-            return
-
-        projects = filter_projects(CLOWDER_CONTROLLER.groups, project_names=self.app.pargs.projects)
+        projects = filter_projects(CLOWDER_CONTROLLER.projects, self.app.pargs.projects)
         for project in projects:
-            run_project_command(project, self.app.pargs.skip, 'diff')
+            print(project.status())
+            project.diff()

src/clowder/cli/forall_controller.py

@@ -6,16 +6,15 @@
 """
 
 import os
-from typing import List, Optional
+from typing import List
 
 from cement.ext.ext_argparse import ArgparseController, expose
 
 from clowder.clowder_controller import CLOWDER_CONTROLLER, ClowderController
 from clowder.clowder_repo import print_clowder_repo_status
 from clowder.util.clowder_utils import (
     filter_projects,
-    options_help_message,
-    run_project_command
+    options_help_message
 )
 from clowder.util.decorators import valid_clowder_yaml_required
 from clowder.util.parallel_commands import forall_parallel
@@ -39,19 +38,10 @@ class Meta:
                                        help='command or script to run in project directories')),
             (['--ignore-errors', '-i'], dict(action='store_true', help='ignore errors in command or script')),
             (['--parallel'], dict(action='store_true', help='run commands in parallel')),
-            (['--groups', '-g'], dict(choices=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      default=CLOWDER_CONTROLLER.get_all_group_names(),
-                                      nargs='+', metavar='GROUP',
-                                      help=options_help_message(CLOWDER_CONTROLLER.get_all_group_names(),
-                                                                'groups to run command for'))),
             (['--projects', '-p'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                        nargs='+', metavar='PROJECT',
+                                        default=['all'], nargs='+', metavar='PROJECT',
                                         help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                                  'projects to run command for'))),
-            (['--skip', '-s'], dict(choices=CLOWDER_CONTROLLER.get_all_project_names(),
-                                    nargs='+', metavar='PROJECT', default=[],
-                                    help=options_help_message(CLOWDER_CONTROLLER.get_all_project_names(),
-                                                              'projects to skip')))
+                                                                  'projects to run command for')))
             ]
     )
     def forall(self) -> None:
@@ -65,32 +55,27 @@ def _forall(self) -> None:
         """Clowder forall command private implementation"""
 
         forall(CLOWDER_CONTROLLER, self.app.pargs.command, self.app.pargs.ignore_errors,
-               group_names=self.app.pargs.groups, project_names=self.app.pargs.projects,
-               skip=self.app.pargs.skip, parallel=self.app.pargs.parallel)
+               project_names=self.app.pargs.projects, parallel=self.app.pargs.parallel)
 
 
 def forall(clowder: ClowderController, command: List[str], ignore_errors: bool,
-           group_names: List[str], project_names: Optional[List[str]] = None,
-           skip: Optional[List[str]] = None, parallel: bool = False) -> None:
+           project_names: List[str], parallel: bool = False) -> None:
     """Runs script in project directories specified
 
     :param ClowderController clowder: ClowderController instance
     :param list[str] command: Command or script and optional arguments
     :param bool ignore_errors: Whether to exit if command returns a non-zero exit code
-    :param list[str] group_names: Group names to run command for
-    :param Optional[List[str]] project_names: Project names to clean
-    :param Optional[List[str]] skip: Project names to skip
+    :param List[str] project_names: Project names to clean
     :param bool parallel: Whether command is being run in parallel, affects output
     """
 
-    skip = [] if skip is None else skip
-
-    projects = filter_projects(clowder.groups, group_names=group_names, project_names=project_names)
+    projects = filter_projects(clowder.projects, project_names)
 
     if parallel:
-        forall_parallel([" ".join(command)], skip, ignore_errors, projects)
+        forall_parallel([" ".join(command)], projects, ignore_errors)
         if os.name == "posix":
             return
 
     for project in projects:
-        run_project_command(project, skip, 'run', [" ".join(command)], ignore_errors)
+        print(project.status())
+        project.run([" ".join(command)], ignore_errors)