Merge pull request #212 from ceph/wip-9117-wusui

Move output in task/s3readwrite

tasks/s3readwrite.py

@@ -9,6 +9,7 @@
 import random
 import string
 import yaml
+from datetime import datetime
 
 from teuthology import misc as teuthology
 from teuthology import contextutil
@@ -88,6 +89,11 @@ def create_users(ctx, config):
     testdir = teuthology.get_testdir(ctx)
     users = {'s3': 'foo'}
     cached_client_user_names = dict()
+    try:
+        archive_dir = ctx.archive
+    except AttributeError:
+        archive_dir = '/tmp'
+    arch_file = os.sep.join([archive_dir, 's3readwrite-{site}-{time}'])
     for client in config['clients']:
         cached_client_user_names[client] = dict()
         s3tests_conf = config['s3tests_conf'][client]
@@ -97,6 +103,11 @@ def create_users(ctx, config):
         s3tests_conf['readwrite'].setdefault('writers', 3)
         s3tests_conf['readwrite'].setdefault('duration', 300)
         s3tests_conf['readwrite'].setdefault('files', {})
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        s3tests_conf['readwrite'].setdefault('output',
+                arch_file.format(site=client, time=timestamp))
+        s3tests_conf['readwrite'].setdefault('archive_dir', archive_dir)
+        s3tests_conf['readwrite'].setdefault('verbose', False) 
         rwconf = s3tests_conf['readwrite']
         rwconf['files'].setdefault('num', 10)
         rwconf['files'].setdefault('size', 2000)
@@ -215,20 +226,28 @@ def run_tests(ctx, config):
     """
     assert isinstance(config, dict)
     testdir = teuthology.get_testdir(ctx)
-    for client, client_config in config.iteritems():
+    rem_file_list = []
+    for client, client_config in config['clients'].iteritems():
         (remote,) = ctx.cluster.only(client).remotes.keys()
         conf = teuthology.get_file(remote, '{tdir}/archive/s3readwrite.{client}.config.yaml'.format(tdir=testdir, client=client))
         args = [
                 '{tdir}/s3-tests/virtualenv/bin/s3tests-test-readwrite'.format(tdir=testdir),
                 ]
         if client_config is not None and 'extra_args' in client_config:
             args.extend(client_config['extra_args'])
-
-        ctx.cluster.only(client).run(
-            args=args,
-            stdin=conf,
-            )
-    yield
+        output_file = client_config['readwrite']['output']
+        archive_dir = client_config['readwrite']['archive_dir']
+        log.info('Output generated by s3readwrite test has been saved in %s' % output_file)
+        ctx.cluster.only(client).run(args=['mkdir', '-p', archive_dir])
+        args.extend([run.Raw('>'), output_file])
+        rem_file_list.append((remote, client, output_file, archive_dir))
+        ctx.cluster.only(client).run(args=args, stdin=StringIO(conf))
+    try:
+        yield
+    finally:
+        for remote_info in rem_file_list:
+            teuthology.pull_directory(remote_info[0], remote_info[3], remote_info[3])
+            ctx.cluster.only(remote_info[1]).run(args=['rm', '-f', remote_info[2]])
 
 
 @contextlib.contextmanager
@@ -271,6 +290,8 @@ def task(ctx, config):
                 readers: 10
                 writers: 3
                 duration: 600
+                output: /tmp/output
+                verbose: True
                 files:
                   num: 10
                   size: 2000
@@ -292,6 +313,8 @@ def task(ctx, config):
                 access_key: myaccesskey
                 secret_key: mysecretkey
 
+    This test writes results to the file specified in the output line under s3readwrite.
+    If no output is specified, the results are saved in a temporary file.
     """
     assert config is None or isinstance(config, list) \
         or isinstance(config, dict), \
@@ -340,7 +363,10 @@ def task(ctx, config):
                 clients=config,
                 s3tests_conf=s3tests_conf,
                 )),
-        lambda: run_tests(ctx=ctx, config=config),
+        lambda: run_tests(ctx=ctx, config=dict(
+                clients=config,
+                s3tests_conf=s3tests_conf,
+                )),
         ):
         pass
     yield