Permalink
Browse files

adding benchmarks testing work

This is based initial work by Pavel which he had
done quickly to generate some initial baseline stats:
https://gist.github.com/1aeaea76519c628702c4

The plan is to test each major operation to get
a sense of how we compare.

python benchmarks/benchmark.py requires that
python-memcached and pylibcb be installed as
they are needed for comparison testing.

Change-Id: I7cca7c5bd9a62a3ad82abe4776eb9f06cd873e23
Reviewed-on: http://review.couchbase.org/19421
Reviewed-by: Benjamin Young <benjamin@couchbase.com>
Tested-by: Benjamin Young <benjamin@couchbase.com>
  • Loading branch information...
1 parent 4fc7145 commit 6d960647a16fa16bb39931c4323e7a402d0faa3a @BigBlueHat BigBlueHat committed with Benjamin Young Aug 8, 2012
Showing with 505 additions and 0 deletions.
  1. +1 −0 .gitignore
  2. +21 −0 README.md
  3. +150 −0 couchbase/benchmarks/benchmark.py
  4. +230 −0 couchbase/benchmarks/profile_it.py
  5. +103 −0 couchbase/benchmarks/profile_stats.py
View
@@ -9,3 +9,4 @@ test.ini
cover
*.egg*
RELEASE-VERSION
+couchbase/benchmarks/profiles
View
@@ -152,3 +152,24 @@ In 1.8.1 use `cbflushctl`:
In 2.0.0 use `cbepctl`:
cbepctl localhost:11210 set flush_param flushall_enabled true
+
+
+BASIC BENCHMARKING
+==================
+
+We like things to go fast, and we can't know how fast they're going
+without measuring them. To check the various Python SDK pieces against
+python-memcached and pylibc, we've created a simple cProfile-based
+performance reporting tool.
+
+To run this (on a *testing* cluster, *not* on dev or production), do:
+
+ python couchbase/benchmarks/benchmark.py
+
+To read the profile output do:
+
+ python couchbase/benchmarks/profiles/{name_of_profile_output_file}
+
+It's early stage stuff as yet, but it should be helpful for quick
+progress comparison, and to help track down places the SDK can improve.
+
@@ -0,0 +1,150 @@
+#
+# Copyright 2012, Couchbase, Inc.
+# All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import random
+import string
+import uuid
+import json
+import time
+import os
+import sys
+
+import cProfile
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+# Benchmark parameters
+HOST = "127.0.0.1"
+ITEMS = 50000 # Total number of documents to create/read
+VALUES_PER_DOC = 1
+VALUE_LENGTH = 92 # together with previous parameter it defines document size
+PROFILES = os.path.join(here, 'profiles')
+
+
+def prepare_data():
+ # Test data
+ print "Preparing data..."
+
+ # Random string generator
+ randstr = lambda length: ''.join(random.choice(string.letters +
+ string.digits)
+ for _ in range(length))
+
+ data = dict()
+ for item in range(ITEMS):
+ key = uuid.uuid4().hex
+ value = dict((randstr(10), randstr(VALUE_LENGTH))
+ for _ in range(VALUES_PER_DOC))
+ data[key] = json.dumps(value)
+
+ return data
+
+
+class Bench:
+ """Base Bench class for handling setup, data prep, and execution"""
+ def __init__(self):
+ """Override this with library integration"""
+ raise NotImplementedError
+
+ def do(self, test_func, data, title):
+ # Insert all items
+ print " " + title + " data..."
+
+ def do_it():
+ start_time = time.time()
+ for key, value in data.items():
+ test_func(key, value)
+ end_time = time.time()
+ print " cmds/sec: {0}".format(ITEMS / (end_time - start_time))
+
+ cProfile.runctx('do_it()', globals(), locals(),
+ os.path.join(PROFILES,
+ str(self.__class__)[9:] + '.' +
+ test_func.__name__))
+
+
+class PythonMemcacheBench(Bench):
+ def __init__(self):
+ import memcache
+ self.client = memcache.Client(["{0}:11210".format(HOST)], debug=0)
+
+ def sets(self, key, value):
+ self.client.set(key, value)
+
+ def gets(self, key, value):
+ self.client.get(key)
+
+
+class MemcachedClientBench(Bench):
+ def __init__(self):
+ from couchbase.memcachedclient import MemcachedClient
+ self.client = MemcachedClient(port=11210)
+
+ def sets(self, key, value):
+ self.client.set(key, 0, 0, value)
+
+ def gets(self, key, value):
+ self.client.get(key)
+
+
+class VBucketAwareClientBench(MemcachedClientBench):
+ def __init__(self):
+ from couchbase.vbucketawareclient import VBucketAwareClient
+ self.client = VBucketAwareClient(port=11210)
+
+
+class CouchbaseClientBench(MemcachedClientBench):
+ def __init__(self):
+ from couchbase.couchbaseclient import CouchbaseClient
+ self.client = \
+ CouchbaseClient('http://{0}:8091/pools/default'.format(HOST),
+ 'default')
+
+
+class CouchbaseBench(MemcachedClientBench):
+ def __init__(self):
+ from couchbase import Couchbase
+ self.client = Couchbase(HOST, 'Administrator', 'asdasd')['default']
+
+
+class PyLibCbBench(Bench):
+ def __init__(self):
+ from pylibcb import Client
+ self.client = Client(host=HOST, user='Administrator', passwd='asdasd',
+ bucket='default')
+
+ def sets(self, key, value):
+ self.client.set(key, value)
+
+ def gets(self, key, value):
+ self.client.get(key)
+
+
+def main():
+ data = prepare_data()
+ for name in ["PythonMemcache", "MemcachedClient",
+ "VBucketAwareClient", "CouchbaseClient", "Couchbase",
+ "PyLibCb"]:
+ print name
+ bench = getattr(sys.modules[__name__], name + 'Bench')()
+ for (doing, func) in [('Set', 'sets'), ('Get', 'gets')]:
+ bench.do(getattr(bench, func), data, doing)
+
+
+if __name__ == "__main__":
+ main()
+ os._exit(0)
Oops, something went wrong.

0 comments on commit 6d96064

Please sign in to comment.