Merge pull request #14487 from zmc/wip-kraken-workunit

qa/tasks/workunit: Backport repo fixes from master

qa/tasks/workunit.py

@@ -5,6 +5,7 @@
 import pipes
 import os
 
+from copy import deepcopy
 from util import get_remote_for_role
 
 from teuthology import misc
@@ -16,6 +17,58 @@
 log = logging.getLogger(__name__)
 
 
+class Refspec:
+    def __init__(self, refspec):
+        self.refspec = refspec
+
+    def __str__(self):
+        return self.refspec
+
+    def _clone(self, git_url, clonedir, opts=None):
+        if opts is None:
+            opts = []
+        return (['rm', '-rf', clonedir] +
+                [run.Raw('&&')] +
+                ['git', 'clone'] + opts +
+                [git_url, clonedir])
+
+    def _cd(self, clonedir):
+        return ['cd', clonedir]
+
+    def _checkout(self):
+        return ['git', 'checkout', self.refspec]
+
+    def clone(self, git_url, clonedir):
+        return (self._clone(git_url, clonedir) +
+                [run.Raw('&&')] +
+                self._cd(clonedir) +
+                [run.Raw('&&')] +
+                self._checkout())
+
+
+class Branch(Refspec):
+    def __init__(self, tag):
+        Refspec.__init__(self, tag)
+
+    def clone(self, git_url, clonedir):
+        opts = ['--depth', '1',
+                '--branch', self.refspec]
+        return (self._clone(git_url, clonedir, opts) +
+                [run.Raw('&&')] +
+                self._cd(clonedir))
+
+
+class Head(Refspec):
+    def __init__(self):
+        Refspec.__init__(self, 'HEAD')
+
+    def clone(self, git_url, clonedir):
+        opts = ['--depth', '1']
+        return (self._clone(git_url, clonedir, opts) +
+                [run.Raw('&&')] +
+                self._cd(clonedir))
+
+
 def task(ctx, config):
     """
     Run ceph on all workunits found under the specified path.
@@ -75,16 +128,24 @@ def task(ctx, config):
     assert isinstance(config.get('clients'), dict), \
         'configuration must contain a dictionary of clients'
 
-    overrides = ctx.config.get('overrides', {})
-    misc.deep_merge(config, overrides.get('workunit', {}))
-
-    refspec = config.get('branch')
-    if refspec is None:
-        refspec = config.get('tag')
+    # mimic the behavior of the "install" task, where the "overrides" are
+    # actually the defaults of that task. in other words, if none of "sha1",
+    # "tag", or "branch" is specified by a "workunit" tasks, we will update
+    # it with the information in the "workunit" sub-task nested in "overrides".
+    overrides = deepcopy(ctx.config.get('overrides', {}).get('workunit', {}))
+    refspecs = {'branch': Branch, 'tag': Refspec, 'sha1': Refspec}
+    if any(map(lambda i: i in config, refspecs.iterkeys())):
+        for i in refspecs.iterkeys():
+            overrides.pop(i, None)
+    misc.deep_merge(config, overrides)
+
+    for spec, cls in refspecs.iteritems():
+        refspec = config.get(spec)
+        if refspec:
+            refspec = cls(refspec)
+            break
     if refspec is None:
-        refspec = config.get('sha1')
-    if refspec is None:
-        refspec = 'HEAD'
+        refspec = Head()
 
     timeout = config.get('timeout', '3h')
 
@@ -276,6 +337,7 @@ def _spawn_on_all_clients(ctx, refspec, tests, env, subdir, timeout=None):
     for role, _ in client_remotes.items():
         _delete_dir(ctx, role, created_mountpoint[role])
 
+
 def _run_tests(ctx, refspec, role, tests, env, subdir=None, timeout=None):
     """
     Run the individual test. Create a scratch directory and then extract the
@@ -308,51 +370,24 @@ def _run_tests(ctx, refspec, role, tests, env, subdir=None, timeout=None):
     clonedir = '{tdir}/clone.{role}'.format(tdir=testdir, role=role)
     srcdir = '{cdir}/qa/workunits'.format(cdir=clonedir)
 
-    git_url = teuth_config.get_ceph_git_url()
+    git_url = teuth_config.get_ceph_qa_suite_git_url()
+    # if we are running an upgrade test, and ceph-ci does not have branches like
+    # `jewel`, so should use ceph.git as an alternative.
     try:
-        remote.run(
-            logger=log.getChild(role),
-            args=[
-                'rm',
-                '-rf',
-                clonedir,
-                run.Raw('&&'),
-                'git',
-                'clone',
-                git_url,
-                clonedir,
-                run.Raw('&&'),
-                'cd', '--', clonedir,
-                run.Raw('&&'),
-                'git', 'checkout', refspec,
-            ],
-        )
+        remote.run(logger=log.getChild(role),
+                   args=refspec.clone(git_url, clonedir))
     except CommandFailedError:
-        alt_git_url = git_url.replace('ceph-ci', 'ceph')
+        if not git_url.endswith('/ceph-ci.git'):
+            raise
+        alt_git_url = git_url.replace('/ceph-ci.git', '/ceph.git')
         log.info(
             "failed to check out '%s' from %s; will also try in %s",
             refspec,
             git_url,
             alt_git_url,
         )
-        remote.run(
-            logger=log.getChild(role),
-            args=[
-                'rm',
-                '-rf',
-                clonedir,
-                run.Raw('&&'),
-                'git',
-                'clone',
-                alt_git_url,
-                clonedir,
-                run.Raw('&&'),
-                'cd', '--', clonedir,
-                run.Raw('&&'),
-                'git', 'checkout', refspec,
-            ],
-        )
-
+        remote.run(logger=log.getChild(role),
+                   args=refspec.clone(alt_git_url, clonedir))
     remote.run(
         logger=log.getChild(role),
         args=[