Merge pull request #24620 from dillaman/wip-36431-luminous

luminous: qa/workunits: replace 'realpath' with 'readlink -f' in fsstress.sh 

Reviewed-by: Jason Dillaman <dillaman@redhat.com>

qa/suites/krbd/unmap/tasks/unmap.yaml

@@ -2,4 +2,4 @@ tasks:
 - cram:
     clients:
       client.0:
-      - http://git.ceph.com/?p={repo};a=blob_plain;hb={branch};f=src/test/cli-integration/rbd/unmap.t
+      - src/test/cli-integration/rbd/unmap.t

qa/suites/rbd/maintenance/qemu/xfstests.yaml

@@ -10,5 +10,5 @@ io_workload:
           type: block
           disks: 3
           time_wait: 120
-          test: http://git.ceph.com/?p={repo};a=blob_plain;hb={branch};f=qa/run_xfstests_qemu.sh
+          test: qa/run_xfstests_qemu.sh
 exclude_arch: armv7l

qa/suites/rbd/openstack/workloads/devstack-tempest-gate.yaml

@@ -7,7 +7,7 @@ tasks:
       disks:
       - image_size: 30720
       - image_size: 30720
-      test: http://git.ceph.com/?p={repo};a=blob_plain;hb={branch};f=qa/workunits/rbd/run_devstack_tempest.sh
+      test: qa/workunits/rbd/run_devstack_tempest.sh
       image_url: https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
       cloud_config_archive:
       - type: text/cloud-config

qa/suites/rbd/qemu/workloads/qemu_bonnie.yaml

@@ -2,5 +2,5 @@ tasks:
 - qemu:
     all:
       clone: true
-      test: http://git.ceph.com/?p={repo};a=blob_plain;hb={branch};f=qa/workunits/suites/bonnie.sh
+      test: qa/workunits/suites/bonnie.sh
 exclude_arch: armv7l

qa/suites/rbd/qemu/workloads/qemu_fsstress.yaml

@@ -2,5 +2,5 @@ tasks:
 - qemu:
     all:
       clone: true
-      test: http://git.ceph.com/?p={repo};a=blob_plain;hb={branch};f=qa/workunits/suites/fsstress.sh
+      test: qa/workunits/suites/fsstress.sh
 exclude_arch: armv7l

qa/suites/rbd/qemu/workloads/qemu_iozone.yaml.disabled

@@ -1,6 +1,6 @@
 tasks:
 - qemu:
     all:
-      test: http://git.ceph.com/?p={repo};a=blob_plain;h={branch};f=qa/workunits/suites/iozone.sh
+      test: qa/workunits/suites/iozone.sh
       image_size: 20480
 exclude_arch: armv7l

qa/suites/rbd/qemu/workloads/qemu_xfstests.yaml

@@ -4,5 +4,5 @@ tasks:
       clone: true
       type: block
       disks: 3
-      test: http://git.ceph.com/?p={repo};a=blob_plain;hb={branch};f=qa/run_xfstests_qemu.sh
+      test: qa/run_xfstests_qemu.sh
 exclude_arch: armv7l

qa/suites/rbd/singleton/all/formatted-output.yaml

@@ -7,4 +7,4 @@ tasks:
 - cram:
     clients:
       client.0:
-      - http://git.ceph.com/?p={repo};a=blob_plain;hb={branch};f=src/test/cli-integration/rbd/formatted-output.t
+      - src/test/cli-integration/rbd/formatted-output.t

qa/tasks/cram.py

@@ -4,6 +4,8 @@
 import logging
 import os
 
+from util.workunit import get_refspec_after_overrides
+
 from teuthology import misc as teuthology
 from teuthology.parallel import parallel
 from teuthology.orchestra import run
@@ -13,7 +15,7 @@
 
 def task(ctx, config):
     """
-    Run all cram tests from the specified urls on the specified
+    Run all cram tests from the specified paths on the specified
     clients. Each client runs tests in parallel.
 
     Limitations:
@@ -27,9 +29,9 @@ def task(ctx, config):
         - cram:
             clients:
               client.0:
-              - http://download.ceph.com/qa/test.t
-              - http://download.ceph.com/qa/test2.t]
-              client.1: [http://download.ceph.com/qa/test.t]
+              - qa/test.t
+              - qa/test2.t]
+              client.1: [qa/test.t]
             branch: foo
 
     You can also run a list of cram tests on all clients::
@@ -38,7 +40,7 @@ def task(ctx, config):
         - ceph:
         - cram:
             clients:
-              all: [http://download.ceph.com/qa/test.t]
+              all: [qa/test.t]
 
     :param ctx: Context
     :param config: Configuration
@@ -52,21 +54,10 @@ def task(ctx, config):
     testdir = teuthology.get_testdir(ctx)
 
     overrides = ctx.config.get('overrides', {})
-    teuthology.deep_merge(config, overrides.get('workunit', {}))
-
-    refspec = config.get('branch')
-    if refspec is None:
-        refspec = config.get('tag')
-    if refspec is None:
-        refspec = config.get('sha1')
-    if refspec is None:
-        refspec = 'HEAD'
-
-    # hack: the git_url is always ceph-ci or ceph
-    git_url = teuth_config.get_ceph_git_url()
-    repo_name = 'ceph.git'
-    if git_url.count('ceph-ci'):
-        repo_name = 'ceph-ci.git'
+    refspec = get_refspec_after_overrides(config, overrides)
+
+    git_url = teuth_config.get_ceph_qa_suite_git_url()
+    log.info('Pulling tests from %s ref %s', git_url, refspec)
 
     try:
         for client, tests in clients.iteritems():
@@ -82,13 +73,14 @@ def task(ctx, config):
                     'install', 'cram==0.6',
                     ],
                 )
+            clone_dir = '{tdir}/clone.{role}'.format(tdir=testdir, role=client)
+            remote.run(args=refspec.clone(git_url, clone_dir))
+
             for test in tests:
-                url = test.format(repo=repo_name, branch=refspec)
-                log.info('fetching test %s for %s', url, client)
                 assert test.endswith('.t'), 'tests must end in .t'
                 remote.run(
                     args=[
-                        'wget', '-nc', '-nv', '-P', client_dir, '--', url,
+                        'cp', '--', os.path.join(clone_dir, test), client_dir,
                         ],
                     )
 
@@ -119,6 +111,7 @@ def task(ctx, config):
                 args=[
                     'rm', '-rf', '--',
                     '{tdir}/virtualenv'.format(tdir=testdir),
+                    clone_dir,
                     run.Raw(';'),
                     'rmdir', '--ignore-fail-on-non-empty', client_dir,
                     ],

qa/tasks/qemu.py

@@ -15,6 +15,8 @@
 from teuthology.orchestra import run
 from teuthology.config import config as teuth_config
 
+from util.workunit import get_refspec_after_overrides
+
 log = logging.getLogger(__name__)
 
 DEFAULT_NUM_DISKS = 2
@@ -102,24 +104,20 @@ def generate_iso(ctx, config):
 
     # use ctx.config instead of config, because config has been
     # through teuthology.replace_all_with_clients()
-    refspec = ctx.config.get('branch')
-    if refspec is None:
-        refspec = ctx.config.get('tag')
-    if refspec is None:
-        refspec = ctx.config.get('sha1')
-    if refspec is None:
-        refspec = 'HEAD'
-
-    # hack: the git_url is always ceph-ci or ceph
-    git_url = teuth_config.get_ceph_git_url()
-    repo_name = 'ceph.git'
-    if git_url.count('ceph-ci'):
-        repo_name = 'ceph-ci.git'
+    refspec = get_refspec_after_overrides(ctx.config, {})
+
+    git_url = teuth_config.get_ceph_qa_suite_git_url()
+    log.info('Pulling tests from %s ref %s', git_url, refspec)
 
     for client, client_config in config.iteritems():
         assert 'test' in client_config, 'You must specify a test to run'
-        test_url = client_config['test'].format(repo=repo_name, branch=refspec)
+        test = client_config['test']
+
         (remote,) = ctx.cluster.only(client).remotes.keys()
+
+        clone_dir = '{tdir}/clone.{role}'.format(tdir=testdir, role=client)
+        remote.run(args=refspec.clone(git_url, clone_dir))
+
         src_dir = os.path.dirname(__file__)
         userdata_path = os.path.join(testdir, 'qemu', 'userdata.' + client)
         metadata_path = os.path.join(testdir, 'qemu', 'metadata.' + client)
@@ -181,11 +179,10 @@ def generate_iso(ctx, config):
 
         test_file = '{tdir}/qemu/{client}.test.sh'.format(tdir=testdir, client=client)
 
-        log.info('fetching test %s for %s', test_url, client)
+        log.info('fetching test %s for %s', test, client)
         remote.run(
             args=[
-                'wget', '-nv', '-O', test_file,
-                test_url,
+                'cp', '--', os.path.join(clone_dir, test), test_file,
                 run.Raw('&&'),
                 'chmod', '755', test_file,
                 ],
@@ -210,11 +207,12 @@ def generate_iso(ctx, config):
             (remote,) = ctx.cluster.only(client).remotes.keys()
             remote.run(
                 args=[
-                    'rm', '-f',
+                    'rm', '-rf',
                     '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
                     os.path.join(testdir, 'qemu', 'userdata.' + client),
                     os.path.join(testdir, 'qemu', 'metadata.' + client),
                     '{tdir}/qemu/{client}.test.sh'.format(tdir=testdir, client=client),
+                    '{tdir}/clone.{client}'.format(tdir=testdir, client=client),
                     ],
                 )
 

qa/tasks/util/workunit.py

@@ -0,0 +1,78 @@
+import copy
+
+from teuthology import misc
+from teuthology.orchestra import run
+
+class Refspec:
+    def __init__(self, refspec):
+        self.refspec = refspec
+
+    def __str__(self):
+        return self.refspec
+
+    def _clone(self, git_url, clonedir, opts=None):
+        if opts is None:
+            opts = []
+        return (['rm', '-rf', clonedir] +
+                [run.Raw('&&')] +
+                ['git', 'clone'] + opts +
+                [git_url, clonedir])
+
+    def _cd(self, clonedir):
+        return ['cd', clonedir]
+
+    def _checkout(self):
+        return ['git', 'checkout', self.refspec]
+
+    def clone(self, git_url, clonedir):
+        return (self._clone(git_url, clonedir) +
+                [run.Raw('&&')] +
+                self._cd(clonedir) +
+                [run.Raw('&&')] +
+                self._checkout())
+
+
+class Branch(Refspec):
+    def __init__(self, tag):
+        Refspec.__init__(self, tag)
+
+    def clone(self, git_url, clonedir):
+        opts = ['--depth', '1',
+                '--branch', self.refspec]
+        return (self._clone(git_url, clonedir, opts) +
+                [run.Raw('&&')] +
+                self._cd(clonedir))
+
+
+class Head(Refspec):
+    def __init__(self):
+        Refspec.__init__(self, 'HEAD')
+
+    def clone(self, git_url, clonedir):
+        opts = ['--depth', '1']
+        return (self._clone(git_url, clonedir, opts) +
+                [run.Raw('&&')] +
+                self._cd(clonedir))
+
+
+def get_refspec_after_overrides(config, overrides):
+    # mimic the behavior of the "install" task, where the "overrides" are
+    # actually the defaults of that task. in other words, if none of "sha1",
+    # "tag", or "branch" is specified by a "workunit" tasks, we will update
+    # it with the information in the "workunit" sub-task nested in "overrides".
+    overrides = copy.deepcopy(overrides.get('workunit', {}))
+    refspecs = {'suite_sha1': Refspec, 'suite_branch': Branch,
+                'sha1': Refspec, 'tag': Refspec, 'branch': Branch}
+    if any(map(lambda i: i in config, refspecs.iterkeys())):
+        for i in refspecs.iterkeys():
+            overrides.pop(i, None)
+    misc.deep_merge(config, overrides)
+
+    for spec, cls in refspecs.iteritems():
+        refspec = config.get(spec)
+        if refspec:
+            refspec = cls(refspec)
+            break
+    if refspec is None:
+        refspec = Head()
+    return refspec

qa/tasks/workunit.py

@@ -6,8 +6,8 @@
 import os
 import re
 
-from copy import deepcopy
 from util import get_remote_for_role
+from util.workunit import get_refspec_after_overrides
 
 from teuthology import misc
 from teuthology.config import config as teuth_config
@@ -17,59 +17,6 @@
 
 log = logging.getLogger(__name__)
 
-
-class Refspec:
-    def __init__(self, refspec):
-        self.refspec = refspec
-
-    def __str__(self):
-        return self.refspec
-
-    def _clone(self, git_url, clonedir, opts=None):
-        if opts is None:
-            opts = []
-        return (['rm', '-rf', clonedir] +
-                [run.Raw('&&')] +
-                ['git', 'clone'] + opts +
-                [git_url, clonedir])
-
-    def _cd(self, clonedir):
-        return ['cd', clonedir]
-
-    def _checkout(self):
-        return ['git', 'checkout', self.refspec]
-
-    def clone(self, git_url, clonedir):
-        return (self._clone(git_url, clonedir) +
-                [run.Raw('&&')] +
-                self._cd(clonedir) +
-                [run.Raw('&&')] +
-                self._checkout())
-
-
-class Branch(Refspec):
-    def __init__(self, tag):
-        Refspec.__init__(self, tag)
-
-    def clone(self, git_url, clonedir):
-        opts = ['--depth', '1',
-                '--branch', self.refspec]
-        return (self._clone(git_url, clonedir, opts) +
-                [run.Raw('&&')] +
-                self._cd(clonedir))
-
-
-class Head(Refspec):
-    def __init__(self):
-        Refspec.__init__(self, 'HEAD')
-
-    def clone(self, git_url, clonedir):
-        opts = ['--depth', '1']
-        return (self._clone(git_url, clonedir, opts) +
-                [run.Raw('&&')] +
-                self._cd(clonedir))
-
-
 def task(ctx, config):
     """
     Run ceph on all workunits found under the specified path.
@@ -140,25 +87,8 @@ def task(ctx, config):
     assert isinstance(config.get('clients'), dict), \
         'configuration must contain a dictionary of clients'
 
-    # mimic the behavior of the "install" task, where the "overrides" are
-    # actually the defaults of that task. in other words, if none of "sha1",
-    # "tag", or "branch" is specified by a "workunit" tasks, we will update
-    # it with the information in the "workunit" sub-task nested in "overrides".
-    overrides = deepcopy(ctx.config.get('overrides', {}).get('workunit', {}))
-    refspecs = {'branch': Branch, 'tag': Refspec, 'sha1': Refspec}
-    if any(map(lambda i: i in config, refspecs.iterkeys())):
-        for i in refspecs.iterkeys():
-            overrides.pop(i, None)
-    misc.deep_merge(config, overrides)
-
-    for spec, cls in refspecs.iteritems():
-        refspec = config.get(spec)
-        if refspec:
-            refspec = cls(refspec)
-            break
-    if refspec is None:
-        refspec = Head()
-
+    overrides = ctx.config.get('overrides', {})
+    refspec = get_refspec_after_overrides(config, overrides)
     timeout = config.get('timeout', '3h')
     cleanup = config.get('cleanup', True)