fix s3atests that are failing for sometime

qa/suites/rgw/hadoop-s3a/s3a-hadoop.yaml

@@ -1,16 +1,16 @@
 machine_type: ovh
 openstack:
 - volumes: # attached to each instance
-    count: 3 
-    size: 10 # GB
+    count: 3
+    size: 20 # GB
 overrides:
     ceph_ansible:
       vars:
         ceph_conf_overrides:
           global:
             osd default pool size: 2
-            osd pool default pg num: 128
-            osd pool default pgp num: 128
+            osd pool default pg num: 8
+            osd pool default pgp num: 8
             debug rgw: 20
             debug ms: 1
         ceph_test: true
@@ -22,11 +22,14 @@ overrides:
         ceph_origin: repository
         ceph_repository: dev
 roles:
-- [mon.a, osd.0, osd.1, osd.2, rgw.0]
+- [mon.a, osd.0, osd.1, osd.2]
 - [osd.3, osd.4, osd.5]
 - [osd.6, osd.7, osd.8]
-- [mgr.x]
+- [mon.b, mgr.x, rgw.0]
+
 tasks:
 - ssh-keys:
 - ceph-ansible:
+- dnsmasq:
+    rgw.0: [s3.ceph.com]
 - s3a-hadoop:

qa/tasks/dnsmasq.py

@@ -51,6 +51,9 @@ def replace_resolv(remote, path):
     misc.write_file(remote, path, "nameserver 127.0.0.1\n")
     try:
         # install it
+        if remote.os.package_type == "rpm":
+            # for centos ovh resolv.conf has immutable attribute set
+            remote.run(args=['sudo', 'chattr', '-i', '/etc/resolv.conf'], check_status=False)
         remote.run(args=['sudo', 'cp', path, '/etc/resolv.conf'])
         yield
     finally:

qa/tasks/s3a_hadoop.py

@@ -20,6 +20,7 @@ def task(ctx, config):
            bucket-name: 's3atest' (default)
            access-key: 'anykey' (uses a default value)
            secret-key: 'secretkey' ( uses a default value)
+           dnsmasq-name: 's3.ceph.com'
     """
     if config is None:
         config = {}
@@ -39,6 +40,7 @@ def task(ctx, config):
     hadoop_ver = config.get('hadoop-version', '2.7.3')
     bucket_name = config.get('bucket-name', 's3atest')
     access_key = config.get('access-key', 'EGAQRD2ULOIFKFSKCT4F')
+    dnsmasq_name = config.get('dnsmasq-name', 's3.ceph.com')
     secret_key = config.get(
         'secret-key',
         'zi816w1vZKfaSM85Cl0BxXTwSLyN7zB4RbTswrGb')
@@ -76,15 +78,15 @@ def task(ctx, config):
             run.Raw(hadoop_rel)
         ]
     )
-    dnsmasq_name = 's3.ceph.com'
     configure_s3a(rgw_node, dnsmasq_name, access_key, secret_key, bucket_name, testdir)
-    setup_dnsmasq(rgw_node, dnsmasq_name)
     fix_rgw_config(rgw_node, dnsmasq_name)
     setup_user_bucket(rgw_node, dnsmasq_name, access_key, secret_key, bucket_name, testdir)
     if hadoop_ver.startswith('2.8'):
         # test all ITtests but skip AWS test using public bucket landsat-pds
         # which is not available from within this test
-        test_options = '-Dit.test=ITestS3A* -Dit.test=\!ITestS3AAWSCredentialsProvider* -Dparallel-tests -Dscale -Dfs.s3a.scale.test.huge.filesize=128M verify'
+        test_options = '-Dit.test=ITestS3A* -Dparallel-tests -Dscale \
+                        -Dfs.s3a.scale.test.timeout=1200 \
+                        -Dfs.s3a.scale.test.huge.filesize=256M verify'
     else:
         test_options = 'test -Dtest=S3a*,TestS3A*'
     try:
@@ -94,10 +96,6 @@ def task(ctx, config):
         log.info("Done s3a testing, Cleaning up")
         for fil in ['apache*', 'hadoop*', 'venv*', 'create*']:
             rgw_node.run(args=['rm', run.Raw('-rf'), run.Raw('{tdir}/{file}'.format(tdir=testdir, file=fil))])
-        # restart and let NM restore original config
-        rgw_node.run(args=['sudo', 'systemctl', 'stop', 'dnsmasq'])
-        rgw_node.run(args=['sudo', 'systemctl', 'restart', 'network.service'], check_status=False)
-        rgw_node.run(args=['sudo', 'systemctl', 'status', 'network.service'], check_status=False)
 
 
 def install_prereq(client):
@@ -120,37 +118,6 @@ def install_prereq(client):
                 )
 
 
-def setup_dnsmasq(client, name):
-    """
-    Setup simple dnsmasq name eg: s3.ceph.com
-    Local RGW host can then be used with whatever name has been setup with.
-    """
-    resolv_conf = "nameserver 127.0.0.1\n"
-    dnsmasq_template = """address=/{name}/{ip_address}
-server=8.8.8.8
-server=8.8.4.4
-""".format(name=name, ip_address=client.ip_address)
-    dnsmasq_config_path = '/etc/dnsmasq.d/ceph'
-    # point resolv.conf to local dnsmasq
-    misc.sudo_write_file(
-        remote=client,
-        path='/etc/resolv.conf',
-        data=resolv_conf,
-    )
-    misc.sudo_write_file(
-        remote=client,
-        path=dnsmasq_config_path,
-        data=dnsmasq_template,
-    )
-    client.run(args=['cat', dnsmasq_config_path])
-    # restart dnsmasq
-    client.run(args=['sudo', 'systemctl', 'restart', 'dnsmasq'])
-    client.run(args=['sudo', 'systemctl', 'status', 'dnsmasq'])
-    time.sleep(5)
-    # verify dns name is set
-    client.run(args=['ping', '-c', '4', name])
-
-
 def fix_rgw_config(client, name):
     """
     Fix RGW config in ceph.conf, we need rgw dns name entry
@@ -182,6 +149,8 @@ def fix_rgw_config(client, name):
     client.run(args=['cat', ceph_conf_path])
     client.run(args=['sudo', 'systemctl', 'restart', 'ceph-radosgw.target'])
     client.run(args=['sudo', 'systemctl', 'status', 'ceph-radosgw.target'])
+    # sleep for daemon to be completely up before creating admin user
+    time.sleep(10)
 
 
 def setup_user_bucket(client, dns_name, access_key, secret_key, bucket_name, testdir):
@@ -258,11 +227,16 @@ def run_s3atest(client, maven_version, testdir, test_options):
     """
     aws_testdir = '{testdir}/hadoop/hadoop-tools/hadoop-aws/'.format(testdir=testdir)
     run_test = '{testdir}/apache-maven-{maven_version}/bin/mvn'.format(testdir=testdir, maven_version=maven_version)
+    # Remove AWS CredentialsProvider tests as it hits public bucket from AWS
+    # better solution is to create the public bucket on local server and test
+    rm_test = 'rm src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java'
     client.run(
         args=[
             'cd',
             run.Raw(aws_testdir),
             run.Raw('&&'),
+            run.Raw(rm_test),
+            run.Raw('&&'),
             run.Raw(run_test),
             run.Raw(test_options)
         ]
@@ -280,6 +254,11 @@ def configure_s3a(client, dns_name, access_key, secret_key, bucket_name, testdir
 <value>{name}</value>
 </property>
 
+<property>
+<name>fs.contract.test.fs.s3a</name>
+<value>s3a://{bucket_name}/</value>
+</property>
+
 <property>
 <name>fs.s3a.connection.ssl.enabled</name>
 <value>false</value>