CBQE-3043: Full Disk Scenario for XDCR

Change-Id: I1ad5c9ef97b7a5ad673ca945a1c8fb1860bd93c2
Reviewed-on: http://review.couchbase.org/55214
Tested-by: buildbot <build@couchbase.com>
Reviewed-by: Ritam Sharma <ritamcouchbase@gmail.com>

conf/py-xdcr-fulldisk.conf

@@ -0,0 +1,8 @@
+#things to be done before running these testcases:
+#scripts/install.py has to be run with init_nodes=False
+#scripts/cbqe3043.py has to be run against the ini file - this script will mount a 20MB partition on the nodes required for the test
+xdcr.uniXDCR.unidirectional:
+    test_disk_full,items=50000,rdirection=unidirection
+
+xdcr.biXDCR.bidirectional:
+    test_disk_full,items=25000,rdirection=bidirection

pytests/xdcr/biXDCR.py

@@ -5,6 +5,7 @@
 from xdcrnewbasetests import XDCRNewBaseTest
 from xdcrnewbasetests import NodeHelper
 from xdcrnewbasetests import Utility, BUCKET_NAME, OPS
+from remote.remote_util import RemoteMachineShellConnection
 
 
 # Assumption that at least 2 nodes on every cluster
@@ -357,3 +358,36 @@ def replication_while_rebooting_a_non_master_src_dest_node(self):
         ClusterOperationHelper.wait_for_ns_servers_or_assert([reboot_node_dest], self, wait_if_warmup=True)
         ClusterOperationHelper.wait_for_ns_servers_or_assert([reboot_node_src], self, wait_if_warmup=True)
         self.verify_results()
+
+    def test_disk_full(self):
+        self.setup_xdcr_and_load()
+        self.verify_results()
+
+        self.sleep(self._wait_timeout)
+
+        zip_file = "%s.zip" % (self._input.param("file_name", "collectInfo"))
+        try:
+            for node in [self.src_master, self.dest_master]:
+                self.shell = RemoteMachineShellConnection(node)
+                self.shell.execute_cbcollect_info(zip_file)
+                if self.shell.extract_remote_info().type.lower() != "windows":
+                    command = "unzip %s" % (zip_file)
+                    output, error = self.shell.execute_command(command)
+                    self.shell.log_command_output(output, error)
+                    if len(error) > 0:
+                        raise Exception("unable to unzip the files. Check unzip command output for help")
+                    cmd = 'grep -R "Approaching full disk warning." cbcollect_info*/'
+                    output, _ = self.shell.execute_command(cmd)
+                else:
+                    cmd = "curl -0 http://{1}:{2}@{0}:8091/diag 2>/dev/null | grep 'Approaching full disk warning.'".format(
+                                                        self.src_master.ip,
+                                                        self.src_master.rest_username,
+                                                        self.src_master.rest_password)
+                    output, _ = self.shell.execute_command(cmd)
+                self.assertNotEquals(len(output), 0, "Full disk warning not generated as expected in %s" % node.ip)
+                self.log.info("Full disk warning generated as expected in %s" % node.ip)
+
+                self.shell.delete_files(zip_file)
+                self.shell.delete_files("cbcollect_info*")
+        except Exception as e:
+            self.log.info(e)

pytests/xdcr/uniXDCR.py

@@ -557,5 +557,35 @@ def test_optimistic_replication(self):
                     else:
                         self.fail("Partial optimistic replication detected!")
 
+    def test_disk_full(self):
+        self.setup_xdcr_and_load()
+        self.verify_results()
 
+        self.sleep(self._wait_timeout)
 
+        zip_file = "%s.zip" % (self._input.param("file_name", "collectInfo"))
+        try:
+            for node in [self.src_master, self.dest_master]:
+                self.shell = RemoteMachineShellConnection(node)
+                self.shell.execute_cbcollect_info(zip_file)
+                if self.shell.extract_remote_info().type.lower() != "windows":
+                    command = "unzip %s" % (zip_file)
+                    output, error = self.shell.execute_command(command)
+                    self.shell.log_command_output(output, error)
+                    if len(error) > 0:
+                        raise Exception("unable to unzip the files. Check unzip command output for help")
+                    cmd = 'grep -R "Approaching full disk warning." cbcollect_info*/'
+                    output, _ = self.shell.execute_command(cmd)
+                else:
+                    cmd = "curl -0 http://{1}:{2}@{0}:8091/diag 2>/dev/null | grep 'Approaching full disk warning.'".format(
+                                                        self.src_master.ip,
+                                                        self.src_master.rest_username,
+                                                        self.src_master.rest_password)
+                    output, _ = self.shell.execute_command(cmd)
+                self.assertNotEquals(len(output), 0, "Full disk warning not generated as expected in %s" % node.ip)
+                self.log.info("Full disk warning generated as expected in %s" % node.ip)
+
+                self.shell.delete_files(zip_file)
+                self.shell.delete_files("cbcollect_info*")
+        except Exception as e:
+            self.log.info(e)

scripts/cbqe3043.py

@@ -0,0 +1,78 @@
+import getopt
+import sys
+import os
+
+sys.path.append('.')
+sys.path.append('lib')
+from lib.remote.remote_util import RemoteMachineShellConnection
+from lib.membase.api.rest_client import RestConnection
+import logger
+import logging.config
+import TestInput
+
+def usage(error=None):
+    print "Please provide INI file"
+    sys.exit(error)
+
+def main():
+    try:
+        (opts, _) = getopt.getopt(sys.argv[1:], 'hi:', [])
+        for o, _ in opts:
+            if o == "-h":
+                usage()
+
+            _input = TestInput.TestInputParser.get_test_input(sys.argv)
+            if not _input.servers:
+                usage("ERROR: no servers specified. Please use the -i parameter.")
+    except IndexError:
+        usage()
+    except getopt.GetoptError, error:
+        usage("ERROR: " + str(error))
+
+    for server in _input.servers:
+        shell = RemoteMachineShellConnection(server)
+        command = "mount | grep '/cbqe3043'"
+        output, error = shell.execute_command(command)
+        if len(error) > 0:
+            raise Exception("Unable to determine if a partition of 20MB already exists on " + server.ip)
+        if len(output) > 0:
+            print "/cbqe3043 partition is already mounted on " + server.ip
+        if len(output) == 0:
+            print "/cbqe3043 not mounted on " + server.ip
+            print "creating /cbqe3043 on " + server.ip
+            command = "rm -rf /cbqe3043; mkdir -p /cbqe3043"
+            output, _ = shell.execute_command(command)
+            if len(output) > 0:
+                raise Exception("Unable to create directory /cbqe3043 on " + server.ip)
+            print "creating /usr/disk-img on " + server.ip
+            command = "rm -rf /usr/disk-img; mkdir -p /usr/disk-img"
+            output, _ = shell.execute_command(command)
+            if len(output) > 0:
+                raise Exception("Unable to create directory /usr/disk-img on " + server.ip)
+            print "creating disk file /usr/disk-img/disk-cbqe3043.ext3 on " + server.ip
+            command = "dd if=/dev/zero of=/usr/disk-img/disk-cbqe3043.ext3 count=40960"
+            output, _ = shell.execute_command(command)
+            if len(output) > 0:
+                raise Exception("Unable to create disk file /usr/disk-img/disk-cbqe3043.ext3 on " + server.ip)
+            print "formatting disk file into an ext3 filesystem on " + server.ip
+            command = "/sbin/mkfs -t ext3 -q /usr/disk-img/disk-cbqe3043.ext3 -F"
+            output, _ = shell.execute_command(command)
+            if len(output) > 0:
+                raise Exception("Unable to format disk file into an ext3 filesystem on " + server.ip)
+            print "mounting /cbqe3043 on" + server.ip
+            command = "mount -o loop,rw,usrquota,grpquota /usr/disk-img/disk-cbqe3043.ext3 /cbqe3043"
+            output, _ = shell.execute_command(command)
+            if len(output) > 0:
+                raise Exception("Unable to mount /cbqe3043 on " + server.ip)
+            print "giving 777 permissions for /cbqe3043 on " + server.ip
+            command = "chmod 777 /cbqe3043"
+            output, _ = shell.execute_command(command)
+            if len(output) > 0:
+                raise Exception("Unable to give permissions for /cbqe3043 on " + server.ip)
+
+        print "setting data path to /cbqe3043 on " + server.ip
+        rest_conn = RestConnection(server)
+        rest_conn.set_data_path(data_path="/cbqe3043")
+
+if __name__ == "__main__":
+    main()