Skip to content

Commit

Permalink
pep8
Browse files Browse the repository at this point in the history 
Change-Id: I5920a6049a4b6b8fda95c0680a284fc0431d3d23
Reviewed-on: http://review.couchbase.org/17330
Reviewed-by: Michael Wiederhold <mike@couchbase.com>
Tested-by: Benjamin Young <benjamin@couchbase.com>
  • Loading branch information
@BigBlueHat
BigBlueHat authored and Benjamin Young committed Jun 20, 2012
1 parent 7debf15 commit 8f5557d
Showing 1 changed file with 39 additions and 24 deletions.
63 changes: 39 additions & 24 deletions cbworkloadgen
View file
Expand Up @@ -28,6 +28,7 @@ import sys
import uuid import uuid
import traceback import traceback



class SharedProgressBar(object): class SharedProgressBar(object):
def __init__(self, number_of_items): def __init__(self, number_of_items):
self.bar = ProgressBar(0, number_of_items, 77) self.bar = ProgressBar(0, number_of_items, 77)
Expand All @@ -48,13 +49,15 @@ class SharedProgressBar(object):
sys.stdout.write(str(self.bar) + '\r') sys.stdout.write(str(self.bar) + '\r')
sys.stdout.flush() sys.stdout.flush()



class SmartLoader(object): class SmartLoader(object):
def __init__(self, options, server, sharedProgressBar, thread_id): def __init__(self, options, server, sharedProgressBar, thread_id):
self._options = options self._options = options
self._server = server self._server = server
self._thread = None self._thread = None
self.shut_down = False self.shut_down = False
self._stats = {"total_time": 0, "max": 0, "min": 1 * 1000 * 1000, "samples": 0, "timeouts": 0} self._stats = {"total_time": 0, "max": 0, "min": 1 * 1000 * 1000,
"samples": 0, "timeouts": 0}
self._bar = sharedProgressBar self._bar = sharedProgressBar
self._thread_id = thread_id self._thread_id = thread_id


Expand All @@ -67,28 +70,29 @@ class SmartLoader(object):
try: try:
options = self._options options = self._options
setPrc = options.ratio setPrc = options.ratio
v = CouchbaseClient(self._server, options.bucket, options.password, options.verbose) v = CouchbaseClient(self._server, options.bucket, options.password,
number_of_items = (int(options.items) / int(options.num_of_threads)) options.verbose)
number_of_items = (int(options.items)
/ int(options.num_of_threads))
value = StringUtil.create_value("*", int(options.value_size)) value = StringUtil.create_value("*", int(options.value_size))
for i in range(0, number_of_items): for i in range(0, number_of_items):
if self.shut_down: if self.shut_down:
print "shut down" print "shut down"
break break
key = "%s-%s" % (options.key_prefix, str(uuid.uuid4())[:5]) key = "%s-%s" % (options.key_prefix, str(uuid.uuid4())[:5])
if random.randint(0,100) < setPrc: if random.randint(0, 100) < setPrc:
if options.load_json: if options.load_json:
document = "\"name\":\"pymc-%s\"" % key document = '{"name": "pymc-%s", "age": %s, \
document = document + ",\"age\":%s" % (random.randint(0, 1000)) "index": %s}'
document = document + ",\"index\":%s" % (i) document = document % (key, random.randint(0, 1000), i)
document = "{" + document + "}"
try: try:
self._profile_before() self._profile_before()
v.set(key, 0, 0, document) v.set(key, 0, 0, document)
self._profile_after() self._profile_after()
except MemcachedTimeoutException: except MemcachedTimeoutException:
self._stats["timeouts"] += 1 self._stats["timeouts"] += 1
except Exception, e: except Exception, e:
print "general exception... ",e print "general exception... ", e
else: else:
try: try:
self._profile_before() self._profile_before()
Expand All @@ -97,7 +101,7 @@ class SmartLoader(object):
except MemcachedTimeoutException: except MemcachedTimeoutException:
self._stats["timeouts"] += 1 self._stats["timeouts"] += 1
except Exception, e: except Exception, e:
print "general exception... ",e print "general exception... ", e
else: else:
try: try:
self._profile_before() self._profile_before()
Expand All @@ -106,7 +110,7 @@ class SmartLoader(object):
except MemcachedTimeoutException: except MemcachedTimeoutException:
self._stats["timeouts"] += 1 self._stats["timeouts"] += 1
except Exception, e: except Exception, e:
print "general exception... ",e print "general exception... ", e
self._bar.update() self._bar.update()
v.done() v.done()
v = None v = None
Expand All @@ -116,18 +120,20 @@ class SmartLoader(object):
v.done() v.done()


def print_stats(self): def print_stats(self):
msg = "Thread %s - average set time : %s seconds , min : %s seconds , max : %s seconds , operation timeouts %s" msg = '''Thread %s - average set time : %s seconds , min : %s seconds,
max : %s seconds , operation timeouts %s'''
if self._stats["samples"]: if self._stats["samples"]:
print msg % (self._thread_id, self._stats["total_time"] / self._stats["samples"], print msg % (self._thread_id,
self._stats["min"], self._stats["max"], self._stats["timeouts"]) self._stats["total_time"] / self._stats["samples"],
self._stats["min"], self._stats["max"],
self._stats["timeouts"])


def wait(self, block=False): def wait(self, block=False):
if block: if block:
self._thread.join() self._thread.join()
else: else:
return not self._thread.isAlive() return not self._thread.isAlive()



def stop(self): def stop(self):
self.shut_down = True self.shut_down = True
if v: if v:
Expand All @@ -149,28 +155,37 @@ class SmartLoader(object):
if __name__ == "__main__": if __name__ == "__main__":
parser = OptionParser() parser = OptionParser()
parser.add_option("-n", "--node", dest="node", parser.add_option("-n", "--node", dest="node",
help="node's ns_server ip:port", metavar="127.0.0.1:8091") help="node's ns_server ip:port",
parser.add_option("-b", "--bucket", dest="bucket", help="which bucket to insert data", metavar="127.0.0.1:8091")
parser.add_option("-b", "--bucket", dest="bucket",
help="which bucket to insert data",
default="default", metavar="default") default="default", metavar="default")
parser.add_option("-p", "--password", dest="password", parser.add_option("-p", "--password", dest="password",
help="bucket password", metavar=" ") help="bucket password", metavar=" ")
parser.add_option("-j", "--json", dest="load_json", help="insert json data", parser.add_option("-j", "--json", dest="load_json",
help="insert json data",
default=False, metavar="True") default=False, metavar="True")
parser.add_option("-v", "--verbose", dest="verbose", help="run in verbose mode", parser.add_option("-v", "--verbose", dest="verbose",
help="run in verbose mode",
default=False, metavar="False") default=False, metavar="False")


parser.add_option("-i", "--items", dest="items", help="number of items to be inserted", parser.add_option("-i", "--items", dest="items",
help="number of items to be inserted",
default=100, metavar="100") default=100, metavar="100")


parser.add_option("-r", "--ratio", dest="ratio", help="set get operation ratio", parser.add_option("-r", "--ratio", dest="ratio",
help="set get operation ratio",
default=90, metavar="90") default=90, metavar="90")


parser.add_option("--size", dest="value_size", help="value size,default is 256 byte", parser.add_option("--size", dest="value_size",
help="value size,default is 256 byte",
default=512, metavar="100") default=512, metavar="100")


parser.add_option("--prefix", dest="key_prefix", parser.add_option("--prefix", dest="key_prefix",
help="prefix to use for memcached keys and json _ids,default is random datetime string", help="prefix to use for memcached keys and json _ids, \
default=str(datetime.datetime.now().microsecond)[:5], metavar="pymc") default is random datetime string",
default=str(datetime.datetime.now().microsecond)[:5],
metavar="pymc")


options, args = parser.parse_args() options, args = parser.parse_args()


Expand Down

0 comments on commit 8f5557d

Please sign in to comment.