fixed all single-node tests

we shouldn't loop over all servers in those tests which are meant
for single nodes.
modified all these tests to use info.memoryQuota instead of
mcdMemoryReserved when creating a bucket

Change-Id: Ia9e49c9b9c1be1e8a5814fb8f1a9a4829da11b0c
Reviewed-on: http://review.couchbase.org/9070
Reviewed-by: Farshid Ghods <farshid.ghods@gmail.com>
Tested-by: Farshid Ghods <farshid.ghods@gmail.com>

conf/py-all-2-0.conf

@@ -22,9 +22,10 @@ createbuckettests.CreateMembaseBucketsTests.test_valid_chars
 createbuckettests.CreateMembaseBucketsTests.test_invalid_chars
 #createbuckettests.CreateMembaseBucketsTests.test_max_buckets
 #createbuckettests.CreateMembaseBucketsTests.test_more_than_max_buckets
+#createbuckettests.CreateMembaseBucketsTests.test_more_than_max_buckets
 setgettests.MembaseBucket.test_value_100b
 expirytests.ExpiryTests.test_expired_keys
-expirytests.ExpiryTests.test_expired_keys_tap
+#expirytests.ExpiryTests.test_expired_keys_tap
 memcapable.SimpleSetMembaseBucketDefaultPort.test_set_pos_int_value_pos_flag_key_never_expired
 memcapable.SimpleSetMembaseBucketDefaultPort.test_set_neg_int_value_pos_flag_key_never_expired
 memcapable.SimpleSetMembaseBucketDefaultPort.test_set_pos_float_value_pos_flag_key_never_expired

conf/py-view.conf

@@ -9,4 +9,8 @@ viewtests.ViewTests.test_get_view_during_1_min_load_10k_working_set
 viewtests.ViewTests.test_get_view_during_1_min_load_100k_working_set
 viewtests.ViewTests.test_get_view_during_5_min_load_10k_working_set
 viewtests.ViewTests.test_get_view_during_5_min_load_100k_working_set
+viewtests.ViewTests.test_view_on_5k_docs_multiple_nodes
+viewtests.ViewTests.test_update_view_on_5k_docs_multiple_nodes
+viewtests.ViewTests.test_view_on_10k_docs_multiple_nodes
+viewtests.ViewTests.test_load_10k_during_rebalance
 #viewtests.ViewTests.test_get_view_during_10_min_load_10k_working_set_1_min_after_load

lib/membase/helper/bucket_helper.py

@@ -37,11 +37,11 @@ def create_multiple_buckets(server, replica, bucket_ram_ratio=(2.0 / 3.0), howma
         log = logger.Logger.get_logger()
         rest = RestConnection(server)
         info = rest.get_nodes_self()
-        if info.mcdMemoryReserved < 450.0:
-            log.error("at least need 450MB mcdMemoryReserved")
+        if info.memoryQuota < 450.0:
+            log.error("at least need 450MB memoryQuota")
             success = False
         else:
-            available_ram = info.mcdMemoryReserved * bucket_ram_ratio
+            available_ram = info.memoryQuota * bucket_ram_ratio
             if available_ram / howmany > 100:
                 bucket_ram = int(available_ram / howmany)
             else:
@@ -87,7 +87,7 @@ def create_bucket(serverInfo, name='default', replica=1, port=11210, test_case=N
         rest = RestConnection(serverInfo)
         if bucket_ram < 0:
             info = rest.get_nodes_self()
-            bucket_ram = info.mcdMemoryReserved * 2 / 3
+            bucket_ram = info.memoryQuota * 2 / 3
 
         if password:
             authType = "sasl"
@@ -139,9 +139,9 @@ def delete_bucket_or_assert(serverInfo, bucket = 'default', test_case = None):
         log.info('deleting existing buckets on {0}'.format(serverInfo))
 
         rest = RestConnection(serverInfo)
-        rest.delete_bucket(bucket)
-
-        log.info('deleted bucket : {0} from {1}'.format(bucket,serverInfo.ip))
+        if RestHelper(rest).bucket_exists(bucket):
+            rest.delete_bucket(bucket)
+            log.info('deleted bucket : {0} from {1}'.format(bucket, serverInfo.ip))
         msg = 'bucket "{0}" was not deleted even after waiting for two minutes'.format(bucket)
         if test_case:
             test_case.assertTrue(BucketOperationHelper.wait_for_bucket_deletion(bucket, rest, 200), msg=msg)

lib/membase/helper/rebalance_helper.py

@@ -68,9 +68,6 @@ def print_taps_from_all_nodes(rest, bucket='default'):
                 client = MemcachedClientHelper.direct_client(node_for_stat, bucket)
                 log.info("getting tap stats.. for {0}".format(node_for_stat.ip))
                 tap_stats = client.stats('tap')
-                if tap_stats:
-                    RebalanceHelper.log_interesting_taps(node_for_stat, tap_stats, log)
-                tap_stats = client.stats()
                 if tap_stats:
                     RebalanceHelper.log_interesting_taps(node_for_stat, tap_stats, log)
                 client.close()

pytests/backuptests.py

@@ -90,7 +90,7 @@ def _test_backup_add_restore_bucket_body(self,
         server = self.master
         rest = RestConnection(server)
         info = rest.get_nodes_self()
-        size = int(info.mcdMemoryReserved * 2.0 / 3.0)
+        size = int(info.memoryQuota * 2.0 / 3.0)
         if bucket == "default":
             rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi)
         else:
@@ -156,7 +156,7 @@ def _test_backup_add_restore_bucket_with_expiration_key(self, replica):
         bucket = "default"
         rest = RestConnection(self.master)
         info = rest.get_nodes_self()
-        size = int(info.mcdMemoryReserved * 2.0 / 3.0)
+        size = int(info.memoryQuota * 2.0 / 3.0)
         rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi, replicaNumber=replica)
         BucketOperationHelper.wait_for_memcached(self.master, bucket)
         client = MemcachedClientHelper.direct_client(self.master.ip, "default")

pytests/drainratetests.py

@@ -26,7 +26,7 @@ def _create_default_bucket(self):
         if not helper.bucket_exists(self.bucket):
             node_ram_ratio = BucketOperationHelper.base_bucket_ratio([self.master])
             info = rest.get_nodes_self()
-            available_ram = info.mcdMemoryReserved * node_ram_ratio
+            available_ram = info.memoryQuota * node_ram_ratio
             rest.create_bucket(bucket=self.bucket, ramQuotaMB=int(available_ram))
             ready = BucketOperationHelper.wait_for_memcached(self.master, self.bucket)
             self.assertTrue(ready, msg="wait_for_memcached failed")

pytests/expirytests.py

@@ -23,62 +23,60 @@ class ExpiryTests(unittest.TestCase):
 
     def setUp(self):
         self.log = logger.Logger.get_logger()
-        self._servers = TestInputSingleton.input.servers
-        ClusterOperationHelper.cleanup_cluster(self._servers)
-        BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
+        self.master = TestInputSingleton.input.servers[0]
+        ClusterOperationHelper.cleanup_cluster([self.master])
+        BucketOperationHelper.delete_all_buckets_or_assert([self.master], self)
 
         self._bucket_name = 'default'
-        # get moxi port
-
-
-        for serverInfo in self._servers:
-            rest = RestConnection(serverInfo)
-            info = rest.get_nodes_self()
-            self._bucket_port = info.moxi
-            rest.init_cluster(username=serverInfo.rest_username,
-                              password=serverInfo.rest_password)
-            rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
-            bucket_ram = info.mcdMemoryReserved * 2 / 3
-            rest.create_bucket(bucket=self._bucket_name,
-                               ramQuotaMB=bucket_ram,
-                               proxyPort=info.memcached)
-            msg = 'create_bucket succeeded but bucket "default" does not exist'
-            self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(self._bucket_name, rest), msg=msg)
-            ready = BucketOperationHelper.wait_for_memcached(serverInfo,self._bucket_name)
-            self.assertTrue(ready, "wait_for_memcached failed")
+
+        serverInfo = self.master
+        rest = RestConnection(serverInfo)
+        info = rest.get_nodes_self()
+        self._bucket_port = info.moxi
+        rest.init_cluster(username=serverInfo.rest_username,
+                          password=serverInfo.rest_password)
+        rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
+        bucket_ram = info.memoryQuota * 2 / 3
+        rest.create_bucket(bucket=self._bucket_name,
+                           ramQuotaMB=bucket_ram,
+                           proxyPort=info.memcached)
+        msg = 'create_bucket succeeded but bucket "default" does not exist'
+        self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(self._bucket_name, rest), msg=msg)
+        ready = BucketOperationHelper.wait_for_memcached(serverInfo,self._bucket_name)
+        self.assertTrue(ready, "wait_for_memcached failed")
 
     # test case to set 1000 keys and verify that those keys are stored
     #e1
     def test_expired_keys(self):
-        for serverInfo in self._servers:
-            client = MemcachedClientHelper.direct_client(serverInfo, self._bucket_name)
-            expirations = [2, 5, 10]
-            for expiry in expirations:
-                testuuid = uuid.uuid4()
-                keys = ["key_%s_%d" % (testuuid, i) for i in range(500)]
-                self.log.info("pushing keys with expiry set to {0}".format(expiry))
-                for key in keys:
-                    try:
-                        client.set(key, expiry, 0, key)
-                    except mc_bin_client.MemcachedError as error:
-                        msg = "unable to push key : {0} to bucket : {1} error : {2}"
-                        self.log.error(msg.format(key, client.vbucketId, error.status))
-                        self.fail(msg.format(key, client.vbucketId, error.status))
-                self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
-                delay = expiry + 5
-                msg = "sleeping for {0} seconds to wait for those items with expiry set to {1} to expire"
-                self.log.info(msg.format(delay, expiry))
-                time.sleep(delay)
-                self.log.info('verifying that all those keys have expired...')
-                for key in keys:
-                    try:
-                        client.get(key=key)
-                        msg = "expiry was set to {0} but key: {1} did not expire after waiting for {2}+ seconds"
-                        self.fail(msg.format(expiry, key, delay))
-                    except mc_bin_client.MemcachedError as error:
-                        self.assertEquals(error.status, 1,
-                                          msg="expected error code {0} but saw error code {1}".format(1, error.status))
-                self.log.info("verified that those keys inserted with expiry set to {0} have expired".format(expiry))
+        serverInfo = self.master
+        client = MemcachedClientHelper.direct_client(serverInfo, self._bucket_name)
+        expirations = [2, 5, 10]
+        for expiry in expirations:
+            testuuid = uuid.uuid4()
+            keys = ["key_%s_%d" % (testuuid, i) for i in range(500)]
+            self.log.info("pushing keys with expiry set to {0}".format(expiry))
+            for key in keys:
+                try:
+                    client.set(key, expiry, 0, key)
+                except mc_bin_client.MemcachedError as error:
+                    msg = "unable to push key : {0} to bucket : {1} error : {2}"
+                    self.log.error(msg.format(key, client.vbucketId, error.status))
+                    self.fail(msg.format(key, client.vbucketId, error.status))
+            self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
+            delay = expiry + 5
+            msg = "sleeping for {0} seconds to wait for those items with expiry set to {1} to expire"
+            self.log.info(msg.format(delay, expiry))
+            time.sleep(delay)
+            self.log.info('verifying that all those keys have expired...')
+            for key in keys:
+                try:
+                    client.get(key=key)
+                    msg = "expiry was set to {0} but key: {1} did not expire after waiting for {2}+ seconds"
+                    self.fail(msg.format(expiry, key, delay))
+                except mc_bin_client.MemcachedError as error:
+                    self.assertEquals(error.status, 1,
+                                      msg="expected error code {0} but saw error code {1}".format(1, error.status))
+            self.log.info("verified that those keys inserted with expiry set to {0} have expired".format(expiry))
 
 
     # MB-3964
@@ -88,7 +86,7 @@ def test_expired_keys(self):
     # 4) do a dump with tap.py and count number of deletes sent (should be 0)
     def test_expired_keys_tap(self):
 
-        server = self._servers[0]
+        server = self.master
         # callback to track deletes
         queue = Queue(maxsize=10000)
         listener = TapListener(queue,server,"CMD_TAP_DELETE")
@@ -138,7 +136,7 @@ def test_expired_keys_tap(self):
 
 
     def tearDown(self):
-        BucketOperationHelper.delete_all_buckets_or_assert(servers=self._servers, test_case=self)
+        BucketOperationHelper.delete_all_buckets_or_assert(servers=[self.master], test_case=self)
 
 
 class TapListener(Thread):

pytests/failovertests.py

@@ -164,7 +164,7 @@ def common_test_body(self, replica, failover_reason, load_ratio):
         rest.init_cluster(username=master.rest_username,
                           password=master.rest_password)
         rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
-        bucket_ram = info.mcdMemoryReserved * 2 / 3
+        bucket_ram = info.memoryQuota * 2 / 3
         rest.create_bucket(bucket='default',
                            ramQuotaMB=bucket_ram,
                            replicaNumber=replica,

pytests/management/autofailovertests.py

@@ -175,7 +175,7 @@ def _cluster_setup(self):
                           password=master.rest_password)
         rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
         rest.reset_autofailover()
-        bucket_ram = info.mcdMemoryReserved * 2 / 3
+        bucket_ram = info.memoryQuota * 2 / 3
         rest.create_bucket(bucket=bucket_name,
                            ramQuotaMB=bucket_ram,
                            proxyPort=info.moxi)
@@ -239,7 +239,7 @@ def common_test_body(self, replica, failover_reason, load_ratio, age, max_nodes)
         rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
         rest.update_autofailover_settings(True, age, max_nodes)
         rest.reset_autofailover()
-        bucket_ram = info.mcdMemoryReserved * 2 / 3
+        bucket_ram = info.memoryQuota * 2 / 3
         rest.create_bucket(bucket=bucket_name,
                            ramQuotaMB=bucket_ram,
                            replicaNumber=replica,

pytests/management/bucketops/multinode/create/createtests.py

@@ -24,7 +24,12 @@ def setUp(self):
         self.assertTrue(self.input, msg="input parameters missing...")
         self.servers = self.input.servers
         self.master = self.servers[0]
-
+        rest = RestConnection(self.master)
+        rest.init_cluster(username=self.master.rest_username,
+                          password=self.master.rest_password)
+        info = rest.get_nodes_self()
+        node_ram_ratio = BucketOperationHelper.base_bucket_ratio(self.servers)
+        rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio))
         BucketOperationHelper.delete_all_buckets_or_assert(servers=self.servers, test_case=self)
         ClusterOperationHelper.cleanup_cluster(servers=self.servers)
         credentials = self.input.membase_settings
@@ -80,16 +85,10 @@ def test_twenty_buckets(self):
     # if buckets = 0 then the test will create max number of buckets
     # if ops = 0 then no load is run against the system
     def test_buckets(self, bucket_count=0, ops=0, max_time=0, replicas=[1]):
-        rest = RestConnection(self.master)
-        info = rest.get_nodes_self()
-        rest.init_cluster(username=self.master.rest_username,
-                          password=self.master.rest_password)
-        rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
         bucket_ram = 100
-
         if not bucket_count:
-            bucket_count = info.mcdMemoryReserved / bucket_ram
-        if bucket_count > info.mcdMemoryReserved / bucket_ram:
+            bucket_count = info.memoryQuota / bucket_ram
+        if bucket_count > info.memoryQuota / bucket_ram:
             self.log.error('node does not have enough capacity for {0} buckets, exiting test'.format(bucket_count))
             return