Merge pull request #427 from JrGoodle/protocol

Remove protocol command line option and add to source

clowder_test/clowder_test/cli/base_controller.py

@@ -103,7 +103,7 @@ def unittests(self):
     def write(self):
         """clowder write tests"""
 
-        cats_scripts = ['./write_herd.sh', './write_protocol.sh', './write_prune.sh', './write_repo.sh', './write_start.sh']
+        cats_scripts = ['./write_herd.sh', './write_prune.sh', './write_repo.sh', './write_start.sh']
         for script in cats_scripts:
             execute_test_command(script, os.path.join(self.path, 'cats'),
                                  parallel=self.app.pargs.parallel,

clowder_test/clowder_test/cli/cats_controller.py

@@ -131,14 +131,6 @@ def link(self):
 
         self._execute_command('./link.sh', self.path)
 
-    @expose(
-        help='Run cats protocol tests'
-    )
-    def protocol(self):
-        """clowder cats protocol tests"""
-
-        self._execute_command('./protocol.sh', self.path)
-
     @expose(
         help='Run cats prune tests'
     )

docs/swift-update-checkout-py.md

@@ -19,7 +19,7 @@ git clone git@github.com:apple/swift.git
 mkdir swift-source
 cd swift-source
 clowder init git@github.com:JrGoodle/swift-clowder.git
-clowder herd --parallel --protocol ssh
+clowder herd --parallel
 ```
 
 ## Checkout Version

src/clowder/cli/herd_controller.py

@@ -40,8 +40,6 @@ class Meta:
         help='Clone and update projects with latest changes',
         arguments=[
             (['--parallel'], dict(action='store_true', help='run commands in parallel')),
-            (['--protocol'], dict(choices=['https', 'ssh'], nargs=1, default=None, metavar='PROTOCOL',
-                                  help='Protocol to clone new repos with')),
             (['--rebase', '-r'], dict(action='store_true', help='use rebase instead of pull')),
             (['--depth', '-d'], dict(default=None, type=int, nargs=1, metavar='DEPTH', help='depth to herd')),
             (['--branch', '-b'], dict(nargs=1, default=None, metavar='BRANCH', help='branch to herd if present')),
@@ -75,11 +73,10 @@ def _herd(self):
         branch = None if self.app.pargs.branch is None else self.app.pargs.branch[0]
         tag = None if self.app.pargs.tag is None else self.app.pargs.tag[0]
         depth = None if self.app.pargs.depth is None else self.app.pargs.depth[0]
-        protocol = None if self.app.pargs.protocol is None else self.app.pargs.protocol[0]
 
         kwargs = {'group_names': self.app.pargs.groups, 'project_names': self.app.pargs.projects,
                   'skip': self.app.pargs.skip, 'branch': branch, 'tag': tag,
-                  'depth': depth, 'rebase': self.app.pargs.rebase, 'protocol': protocol}
+                  'depth': depth, 'rebase': self.app.pargs.rebase}
 
         if self.app.pargs.parallel:
             herd_parallel(CLOWDER_CONTROLLER, **kwargs)
@@ -92,7 +89,7 @@ def _herd(self):
 def herd(clowder, group_names, **kwargs):
     """Clone projects or update latest from upstream
 
-    .. py:function:: herd(clowder, group_names, branch=None, tag=None, depth=0, rebase=False, project_names=None, skip=[], protocol=None)
+    .. py:function:: herd(clowder, group_names, branch=None, tag=None, depth=0, rebase=False, project_names=None, skip=[])
 
     :param ClowderController clowder: ClowderController instance
     :param list[str] group_names: Group names to herd
@@ -101,7 +98,6 @@ def herd(clowder, group_names, **kwargs):
         branch (str): Branch to attempt to herd
         tag (str): Tag to attempt to herd
         depth (int): Git clone depth. 0 indicates full clone, otherwise must be a positive integer
-        protocol (str): Git protocol ('ssh' or 'https')
         rebase (bool): Whether to use rebase instead of pulling latest changes
         project_names (list[str]) project_names: Project names to herd
         skip (list[str]): Project names to skip
@@ -113,18 +109,17 @@ def herd(clowder, group_names, **kwargs):
     tag = kwargs.get('tag', None)
     depth = kwargs.get('depth', None)
     rebase = kwargs.get('rebase', False)
-    protocol = kwargs.get('protocol', None)
 
     if project_names is None:
         groups = filter_groups(clowder.groups, group_names)
         validate_groups(groups)
         for group in groups:
             run_group_command(group, skip, 'herd', branch=branch, tag=tag,
-                              depth=depth, rebase=rebase, protocol=protocol)
+                              depth=depth, rebase=rebase)
         return
 
     projects = filter_projects(clowder.groups, project_names=project_names)
     validate_projects(projects)
     for project in projects:
         run_project_command(project, skip, 'herd', branch=branch, tag=tag,
-                            depth=depth, rebase=rebase, protocol=protocol)
+                            depth=depth, rebase=rebase)

src/clowder/cli/sync_controller.py

@@ -39,8 +39,6 @@ class Meta:
                                         nargs='+', metavar='PROJECT',
                                         help=options_help_message(CLOWDER_CONTROLLER.get_all_fork_project_names(),
                                                                   'projects to sync'))),
-            (['--protocol'], dict(choices=['https', 'ssh'], nargs=1, default=None, metavar='PROTOCOL',
-                                  help='Protocol to clone new repos with')),
             (['--rebase', '-r'], dict(action='store_true', help='use rebase instead of pull')),
             (['--parallel'], dict(action='store_true', help='run commands in parallel'))
         ]
@@ -56,34 +54,31 @@ def sync(self):
     def _sync(self):
         """Clowder sync command private implementation"""
 
-        protocol = None if self.app.pargs.protocol is None else self.app.pargs.protocol[0]
-
         all_fork_projects = CLOWDER_CONTROLLER.get_all_fork_project_names()
         if all_fork_projects == '':
             cprint(' - No forks to sync\n', 'red')
             return
-        sync(CLOWDER_CONTROLLER, all_fork_projects, protocol=protocol,
+        sync(CLOWDER_CONTROLLER, all_fork_projects,
              rebase=self.app.pargs.rebase,
              parallel=self.app.pargs.parallel)
 
 
-def sync(clowder, project_names, protocol, rebase=False, parallel=False):
+def sync(clowder, project_names, rebase=False, parallel=False):
     """Sync projects
 
-    .. py:function:: sync(clowder, project_names, protocol, rebase=False, parallel=False)
+    .. py:function:: sync(clowder, project_names, rebase=False, parallel=False)
 
     :param ClowderController clowder: ClowderController instance
     :param list[str] project_names: Project names to sync
-    :param str protocol: Git protocol, 'ssh' or 'https'
     :param Optional[bool] rebase: Whether to use rebase instead of pulling latest changes
     :param Optional[bool] parallel: Whether command is being run in parallel, affects output
     """
 
     projects = filter_projects(clowder.groups, project_names=project_names)
     if parallel:
-        sync_parallel(projects, protocol, rebase=rebase)
+        sync_parallel(projects, rebase=rebase)
         if os.name == "posix":
             return
 
     for project in projects:
-        project.sync(protocol, rebase=rebase)
+        project.sync(rebase=rebase)

src/clowder/clowder_controller.py

@@ -134,7 +134,7 @@ def _load_yaml(self):
         try:
             yaml = load_yaml()
             self.defaults = Defaults(yaml['defaults'])
-            self.sources = [Source(s) for s in yaml['sources']]
+            self.sources = [Source(s, self.defaults) for s in yaml['sources']]
             for group in yaml['groups']:
                 self.groups.append(Group(group, self.defaults, self.sources))
         except (AttributeError, TypeError):

src/clowder/git/project_repo_impl.py

@@ -5,6 +5,7 @@
 
 """
 
+import errno
 import os
 
 from git import Repo, GitError
@@ -429,7 +430,7 @@ def _init_repo(self):
                 try:
                     os.makedirs(self.repo_path)
                 except OSError as err:
-                    if err.errno != os.errno.EEXIST:
+                    if err.errno != errno.EEXIST:
                         raise
             self.repo = Repo.init(self.repo_path)
         except GitError as err:

src/clowder/model/fork.py

@@ -11,6 +11,7 @@
 
 from clowder import ROOT_DIR
 from clowder.git.project_repo import ProjectRepo
+from clowder.git.project_repo_recursive import ProjectRepoRecursive
 from clowder.git.util import (
     existing_git_repository,
     format_project_ref_string,
@@ -24,23 +25,31 @@ class Fork(object):
 
     :ivar str name: Project name
     :ivar str path: Project relative path
-    :ivar str remote_name: Git remote name
+    :ivar str remote: Git remote name
     """
 
-    def __init__(self, fork, path, source, protocol):
+    def __init__(self, fork, path, project_source, sources, project_ref, recursive):
         """Project __init__
 
         :param dict fork: Parsed YAML python object for fork
         :param str path: Fork relative path
-        :param Source source: Source instance
-        :param str protocol: Git protocol ('ssh' or 'https')
+        :param Source project_source: Source instance from project
+        :param list[Source] sources: List of Source instances
+        :param str project_ref: Git ref from project
+        :param bool recursive: Whether to handle submodules
         """
 
         self.path = path
         self.name = fork['name']
-        self.remote_name = fork['remote']
-        self._source = source
-        self._protocol = protocol
+        self.remote = fork['remote']
+        self._ref = fork.get('ref', project_ref)
+        self._recursive = recursive
+
+        self._source = None
+        source_name = fork.get('source', project_source.name)
+        for s in sources:
+            if s.name == source_name:
+                self._source = s
 
     def full_path(self):
         """Return full path to project
@@ -58,7 +67,19 @@ def get_yaml(self):
         :rtype: dict
         """
 
-        return {'name': self.name, 'remote': self.remote_name}
+        return {'name': self.name, 'remote': self.remote}
+
+    def repo(self, **kwargs):
+        """Return ProjectRepo or ProjectRepoRecursive instance
+
+        Keyword Args:
+            parallel (bool): Whether command is being run in parallel
+            print_output (bool): Whether to print output
+        """
+
+        if self._recursive:
+            return ProjectRepoRecursive(self.full_path(), self.remote, self._ref, **kwargs)
+        return ProjectRepo(self.full_path(), self.remote, self._ref, **kwargs)
 
     def status(self):
         """Return formatted fork status
@@ -70,18 +91,12 @@ def status(self):
         if not existing_git_repository(self.path):
             return colored(self.path, 'green')
 
-        repo = ProjectRepo(self.full_path(), self.remote_name, 'refs/heads/master')
+        repo = ProjectRepo(self.full_path(), self.remote, self._ref)
         project_output = format_project_string(repo, self.path)
         current_ref_output = format_project_ref_string(repo)
         return project_output + ' ' + current_ref_output
 
-    def url(self, protocol):
-        """Return project url
-
-        :param str protocol: Git protocol ('ssh' or 'https')
-        """
-
-        if protocol:
-            return git_url(protocol, self._source.url, self.name)
+    def url(self):
+        """Return project url"""
 
-        return git_url(self._protocol, self._source.url, self.name)
+        return git_url(self._source.protocol, self._source.url, self.name)

src/clowder/model/group.py

@@ -18,7 +18,7 @@ class Group(object):
     :ivar bool recursive: Group recursive default
     :ivar str timestamp_author: Group timestamp author default
     :ivar str ref: Group ref default
-    :ivar str remote_name: Group remote name default
+    :ivar str remote: Group remote name default
     :ivar Source source: Group source default
     :ivar list[Project] projects: List of group's projects
     """
@@ -36,7 +36,7 @@ def __init__(self, group, defaults, sources):
         self.recursive = group.get('recursive', defaults.recursive)
         self.timestamp_author = group.get('timestamp_author', defaults.timestamp_author)
         self.ref = group.get('ref', defaults.ref)
-        self.remote_name = group.get('remote', defaults.remote)
+        self.remote = group.get('remote', defaults.remote)
         source_name = group.get('source', defaults.source)
 
         for source in sources:
@@ -84,7 +84,7 @@ def get_yaml(self, resolved=False):
                  'depth': self.depth,
                  'ref': self.ref,
                  'recursive': self.recursive,
-                 'remote': self.remote_name,
+                 'remote': self.remote,
                  'source': self.source.name,
                  'projects': [p.get_yaml(resolved=True) for p in self.projects]}