Merge pull request #30069 from smithfarm/wip-40124-mimic

mimic: qa/rgw: don't use ceph-ansible in s3a-hadoop suite

Reviewed-by: Casey Bodley <cbodley@redhat.com>

qa/suites/rgw/hadoop-s3a/clusters/.qa

@@ -0,0 +1 @@
+../.qa/

qa/suites/rgw/hadoop-s3a/clusters/fixed-2.yaml

@@ -0,0 +1 @@
+.qa/clusters/fixed-2.yaml

qa/suites/rgw/hadoop-s3a/s3a-hadoop.yaml

@@ -1,35 +1,11 @@
-machine_type: ovh
-openstack:
-- volumes: # attached to each instance
-    count: 3
-    size: 20 # GB
-overrides:
-    ceph_ansible:
-      vars:
-        ceph_conf_overrides:
-          global:
-            osd default pool size: 2
-            osd pool default pg num: 8
-            osd pool default pgp num: 8
-            debug rgw: 20
-            debug ms: 1
-        ceph_test: true
-        journal_collocation: true
-        osd_auto_discovery: false
-        journal_size: 1024
-        ceph_stable_release: luminous
-        osd_scenario: collocated
-        ceph_origin: repository
-        ceph_repository: dev
-roles:
-- [mon.a, osd.0, osd.1, osd.2]
-- [osd.3, osd.4, osd.5]
-- [osd.6, osd.7, osd.8]
-- [mon.b, mgr.x, rgw.0]
-
 tasks:
+- install:
+- ceph:
 - ssh-keys:
-- ceph-ansible:
 - dnsmasq:
-    rgw.0: [s3.ceph.com]
+    client.0: [s3.]
+- rgw:
+    client.0:
+      dns-name: s3.
 - s3a-hadoop:
+    role: client.0

qa/suites/rgw/multisite/tasks/test_multi.yaml

@@ -7,12 +7,16 @@ tasks:
 - ceph: {cluster: c2}
 - rgw:
     c1.client.0:
+      port: 8000
       valgrind: [--tool=memcheck, --max-threads=1024] # http://tracker.ceph.com/issues/25214
     c1.client.1:
+      port: 8001
       valgrind: [--tool=memcheck, --max-threads=1024]
     c2.client.0:
+      port: 8000
       valgrind: [--tool=memcheck, --max-threads=1024]
     c2.client.1:
+      port: 8001
       valgrind: [--tool=memcheck, --max-threads=1024]
 - rgw-multisite:
 - rgw-multisite-tests:

qa/tasks/ragweed.py

@@ -340,7 +340,7 @@ def task(ctx, config):
                 'rgw':
                     {
                     'port'      : endpoint.port,
-                    'is_secure' : 'yes' if endpoint.cert else 'no',
+                    'is_secure' : endpoint.cert is not None,
                     },
                 'fixtures' : {},
                 'user system'  : {},

qa/tasks/rgw.py

@@ -22,10 +22,12 @@
 log = logging.getLogger(__name__)
 
 class RGWEndpoint:
-    def __init__(self, hostname=None, port=None, cert=None):
+    def __init__(self, hostname=None, port=None, cert=None, dns_name=None, website_dns_name=None):
         self.hostname = hostname
         self.port = port
         self.cert = cert
+        self.dns_name = dns_name
+        self.website_dns_name = website_dns_name
 
     def url(self):
         proto = 'https' if self.cert else 'http'
@@ -105,6 +107,11 @@ def start_rgw(ctx, config, clients):
                                                 kport=keystone_port),
                 ])
 
+        if client_config.get('dns-name'):
+            rgw_cmd.extend(['--rgw-dns-name', endpoint.dns_name])
+        if client_config.get('dns-s3website-name'):
+            rgw_cmd.extend(['--rgw-dns-s3website-name', endpoint.website_dns_name])
+
         rgw_cmd.extend([
             '--foreground',
             run.Raw('|'),
@@ -161,12 +168,7 @@ def start_rgw(ctx, config, clients):
                 )
 
 def assign_endpoints(ctx, config, default_cert):
-    """
-    Assign port numbers starting with port 7280.
-    """
-    port = 7280
     role_endpoints = {}
-
     for role, client_config in config.iteritems():
         client_config = client_config or {}
         remote = get_remote_for_role(ctx, role)
@@ -182,8 +184,19 @@ def assign_endpoints(ctx, config, default_cert):
         else:
             ssl_certificate = None
 
-        role_endpoints[role] = RGWEndpoint(remote.hostname, port, ssl_certificate)
-        port += 1
+        port = client_config.get('port', 443 if ssl_certificate else 80)
+
+        # if dns-name is given, use it as the hostname (or as a prefix)
+        dns_name = client_config.get('dns-name', '')
+        if len(dns_name) == 0 or dns_name.endswith('.'):
+            dns_name += remote.hostname
+
+        website_dns_name = client_config.get('dns-s3website-name')
+        if website_dns_name:
+            if len(website_dns_name) == 0 or website_dns_name.endswith('.'):
+                website_dns_name += remote.hostname
+
+        role_endpoints[role] = RGWEndpoint(remote.hostname, port, ssl_certificate, dns_name, website_dns_name)
 
     return role_endpoints
 

qa/tasks/rgw_logsocket.py

@@ -112,9 +112,10 @@ def task(ctx, config):
             client.1:
               extra_args: ['--exclude', 'test_100_continue']
     """
+    assert hasattr(ctx, 'rgw'), 'rgw-logsocket must run after the rgw task'
     assert config is None or isinstance(config, list) \
         or isinstance(config, dict), \
-        "task s3tests only supports a list or dictionary for configuration"
+        "task rgw-logsocket only supports a list or dictionary for configuration"
     all_clients = ['client.{id}'.format(id=id_)
                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
     if config is None:
@@ -132,13 +133,16 @@ def task(ctx, config):
 
     s3tests_conf = {}
     for client in clients:
+        endpoint = ctx.rgw.role_endpoints.get(client)
+        assert endpoint, 'rgw-logsocket: no rgw endpoint for {}'.format(client)
+
         s3tests_conf[client] = ConfigObj(
             indent_type='',
             infile={
                 'DEFAULT':
                     {
-                    'port'      : 7280,
-                    'is_secure' : 'no',
+                    'port'      : endpoint.port,
+                    'is_secure' : endpoint.cert is not None,
                     },
                 'fixtures' : {},
                 's3 main'  : {},

qa/tasks/s3a_hadoop.py

@@ -20,27 +20,31 @@ def task(ctx, config):
            bucket-name: 's3atest' (default)
            access-key: 'anykey' (uses a default value)
            secret-key: 'secretkey' ( uses a default value)
-           dnsmasq-name: 's3.ceph.com'
+           role: client.0
     """
     if config is None:
         config = {}
 
     assert isinstance(config, dict), \
         "task only supports a dictionary for configuration"
 
+    assert hasattr(ctx, 'rgw'), 's3a-hadoop must run after the rgw task'
+
     overrides = ctx.config.get('overrides', {})
     misc.deep_merge(config, overrides.get('s3a-hadoop', {}))
     testdir = misc.get_testdir(ctx)
-    rgws = ctx.cluster.only(misc.is_type('rgw'))
-    # use the first rgw node to test s3a
-    rgw_node = rgws.remotes.keys()[0]
+
+    role = config.get('role')
+    (remote,) = ctx.cluster.only(role).remotes.keys()
+    endpoint = ctx.rgw.role_endpoints.get(role)
+    assert endpoint, 's3tests: no rgw endpoint for {}'.format(role)
+
     # get versions
     maven_major = config.get('maven-major', 'maven-3')
     maven_version = config.get('maven-version', '3.3.9')
     hadoop_ver = config.get('hadoop-version', '2.7.3')
     bucket_name = config.get('bucket-name', 's3atest')
     access_key = config.get('access-key', 'EGAQRD2ULOIFKFSKCT4F')
-    dnsmasq_name = config.get('dnsmasq-name', 's3.ceph.com')
     secret_key = config.get(
         'secret-key',
         'zi816w1vZKfaSM85Cl0BxXTwSLyN7zB4RbTswrGb')
@@ -52,8 +56,8 @@ def task(ctx, config):
         '{maven_major}/{maven_version}/binaries/'.format(maven_major=maven_major, maven_version=maven_version) + apache_maven
     hadoop_git = 'https://github.com/apache/hadoop'
     hadoop_rel = 'hadoop-{ver} rel/release-{ver}'.format(ver=hadoop_ver)
-    install_prereq(rgw_node)
-    rgw_node.run(
+    install_prereq(remote)
+    remote.run(
         args=[
             'cd',
             testdir,
@@ -78,9 +82,8 @@ def task(ctx, config):
             run.Raw(hadoop_rel)
         ]
     )
-    configure_s3a(rgw_node, dnsmasq_name, access_key, secret_key, bucket_name, testdir)
-    fix_rgw_config(rgw_node, dnsmasq_name)
-    setup_user_bucket(rgw_node, dnsmasq_name, access_key, secret_key, bucket_name, testdir)
+    configure_s3a(remote, endpoint.dns_name, access_key, secret_key, bucket_name, testdir)
+    setup_user_bucket(remote, endpoint.dns_name, access_key, secret_key, bucket_name, testdir)
     if hadoop_ver.startswith('2.8'):
         # test all ITtests but skip AWS test using public bucket landsat-pds
         # which is not available from within this test
@@ -90,12 +93,12 @@ def task(ctx, config):
     else:
         test_options = 'test -Dtest=S3a*,TestS3A*'
     try:
-        run_s3atest(rgw_node, maven_version, testdir, test_options)
+        run_s3atest(remote, maven_version, testdir, test_options)
         yield
     finally:
         log.info("Done s3a testing, Cleaning up")
         for fil in ['apache*', 'hadoop*', 'venv*', 'create*']:
-            rgw_node.run(args=['rm', run.Raw('-rf'), run.Raw('{tdir}/{file}'.format(tdir=testdir, file=fil))])
+            remote.run(args=['rm', run.Raw('-rf'), run.Raw('{tdir}/{file}'.format(tdir=testdir, file=fil))])
 
 
 def install_prereq(client):
@@ -118,41 +121,6 @@ def install_prereq(client):
                 )
 
 
-def fix_rgw_config(client, name):
-    """
-    Fix RGW config in ceph.conf, we need rgw dns name entry
-    and also modify the port to use :80 for s3a tests to work
-    """
-    rgw_dns_name = 'rgw dns name = {name}'.format(name=name)
-    ceph_conf_path = '/etc/ceph/ceph.conf'
-    # append rgw_dns_name
-    client.run(
-        args=[
-            'sudo',
-            'sed',
-            run.Raw('-i'),
-            run.Raw("'/client.rgw*/a {rgw_name}'".format(rgw_name=rgw_dns_name)),
-            ceph_conf_path
-
-        ]
-    )
-    # listen on port 80
-    client.run(
-        args=[
-            'sudo',
-            'sed',
-            run.Raw('-i'),
-            run.Raw('s/:8080/:80/'),
-            ceph_conf_path
-        ]
-    )
-    client.run(args=['cat', ceph_conf_path])
-    client.run(args=['sudo', 'systemctl', 'restart', 'ceph-radosgw.target'])
-    client.run(args=['sudo', 'systemctl', 'status', 'ceph-radosgw.target'])
-    # sleep for daemon to be completely up before creating admin user
-    time.sleep(10)
-
-
 def setup_user_bucket(client, dns_name, access_key, secret_key, bucket_name, testdir):
     """
     Create user with access_key and secret_key that will be

qa/tasks/s3readwrite.py

@@ -293,9 +293,10 @@ def task(ctx, config):
                 secret_key: mysecretkey
 
     """
+    assert hasattr(ctx, 'rgw'), 's3readwrite must run after the rgw task'
     assert config is None or isinstance(config, list) \
         or isinstance(config, dict), \
-        "task s3tests only supports a list or dictionary for configuration"
+        "task s3readwrite only supports a list or dictionary for configuration"
     all_clients = ['client.{id}'.format(id=id_)
                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
     if config is None:
@@ -319,12 +320,14 @@ def task(ctx, config):
             config[client] = {}
         config[client].setdefault('s3', {})
         config[client].setdefault('readwrite', {})
+        endpoint = ctx.rgw.role_endpoints.get(client)
+        assert endpoint, 's3readwrite: no rgw endpoint for {}'.format(client)
 
         s3tests_conf[client] = ({
                 'DEFAULT':
                     {
-                    'port'      : 7280,
-                    'is_secure' : False,
+                    'port'      : endpoint.port,
+                    'is_secure' : endpoint.cert is not None,
                     },
                 'readwrite' : config[client]['readwrite'],
                 's3'  : config[client]['s3'],

qa/tasks/s3roundtrip.py

@@ -262,9 +262,10 @@ def task(ctx, config):
                 secret_key: mysecretkey
 
     """
+    assert hasattr(ctx, 'rgw'), 's3roundtrip must run after the rgw task'
     assert config is None or isinstance(config, list) \
         or isinstance(config, dict), \
-        "task s3tests only supports a list or dictionary for configuration"
+        "task s3roundtrip only supports a list or dictionary for configuration"
     all_clients = ['client.{id}'.format(id=id_)
                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
     if config is None:
@@ -280,11 +281,14 @@ def task(ctx, config):
         config[client].setdefault('s3', {})
         config[client].setdefault('roundtrip', {})
 
+        endpoint = ctx.rgw.role_endpoints.get(client)
+        assert endpoint, 's3roundtrip: no rgw endpoint for {}'.format(client)
+
         s3tests_conf[client] = ({
                 'DEFAULT':
                     {
-                    'port'      : 7280,
-                    'is_secure' : False,
+                    'port'      : endpoint.port,
+                    'is_secure' : endpoint.cert is not None,
                     },
                 'roundtrip' : config[client]['roundtrip'],
                 's3'  : config[client]['s3'],

qa/tasks/s3tests.py

@@ -395,7 +395,7 @@ def task(ctx, config):
                 'DEFAULT':
                     {
                     'port'      : endpoint.port,
-                    'is_secure' : 'yes' if endpoint.cert else 'no',
+                    'is_secure' : endpoint.cert is not None,
                     'api_name'  : 'default',
                     },
                 'fixtures' : {},

qa/workunits/rgw/s3_bucket_quota.pl

@@ -53,7 +53,7 @@ =head1 ARGUMENTS
 my $kruft;
 my $s3;
 my $hostdom  = $ENV{RGW_FQDN}||hostfqdn();
-my $port     = $ENV{RGW_PORT}||7280;
+my $port     = $ENV{RGW_PORT}||80;
 our $hostname = "$hostdom:$port";
 our $testfileloc;
 my $rgw_user = "qa_user";

qa/workunits/rgw/s3_multipart_upload.pl

@@ -48,7 +48,7 @@ =head1 ARGUMENTS
 #== local variables ===
 my $s3;
 my $hostdom  = $ENV{RGW_FQDN}||hostfqdn();
-my $port     = $ENV{RGW_PORT}||7280;
+my $port     = $ENV{RGW_PORT}||80;
 our $hostname = "$hostdom:$port";
 our $testfileloc;
 our $mytestfilename;

qa/workunits/rgw/s3_user_quota.pl

@@ -52,7 +52,7 @@ =head1 ARGUMENTS
 my $kruft;
 my $s3;
 my $hostdom  = $ENV{RGW_FQDN}||hostfqdn();
-my $port     = $ENV{RGW_PORT}||7280;
+my $port     = $ENV{RGW_PORT}||80;
 our $hostname = "$hostdom:$port";
 our $testfileloc;
 our $cnt;