Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Prepare makefile to build workload-generator

  • Loading branch information...
commit 40eb47308317173bb430606fdcb6286fd65d9cc3 1 parent e049532
@bcui6611 bcui6611 authored
View
28 Makefile.am
@@ -0,0 +1,28 @@
+COUCHBASE_PYTHON_LIB = ../couchbase-python-client/couchbase
+
+default:
+
+pythonlibdir=$(libdir)/python
+
+pythonlib_SCRIPTS= workload-generator
+
+PYTHON_TOOLS= wrapper/workload-generator
+
+${PYTHON_TOOLS}: wrapper/wrapper
+ cp $< $@
+
+CLEANFILES = ${PYTHON_TOOLS}
+bin_SCRIPTS = ${PYTHON_TOOLS}
+
+EXTRA_DIST = $(pythonlib_SCRIPTS)
+
+install-data-hook:
+ if test ! "$(mkdir_p)" = ""; then \
+ $(mkdir_p) "$(DESTDIR)$(localstatedatdir)"; \
+ else \
+ echo "WARNING: You may have to create these directories by hand."; \
+ mkdir -p "$(DESTDIR)$(localstatedatdir)"; \
+ fi
+ cp -rf $(COUCHBASE_PYTHON_LIB) $(pythonlibdir)
+
+
View
81 config/autorun.sh
@@ -0,0 +1,81 @@
+#!/bin/sh
+
+AUTOMAKE_FLAGS="--add-missing --copy --force --foreign --warning=portability"
+ACLOCAL_FLAGS="-I m4"
+AUTOCONF_CLAGS="--warnings=error --force"
+
+ARGV0=$0
+ARGS="$@"
+
+die() { echo "$@"; exit 1; }
+
+run() {
+ echo "$ARGV0: running \`$@' $ARGS"
+ $@ $ARGS
+}
+
+# Try to locate a program by using which, and verify that the file is an
+# executable
+locate_binary() {
+ for f in $@
+ do
+ file=`which $f 2>/dev/null | grep -v '^no '`
+ if test -n "$file" -a -x "$file"; then
+ echo $file
+ return 0
+ fi
+ done
+
+ echo ""
+ return 1
+}
+
+if test -f config/pre_hook.sh
+then
+ . config/pre_hook.sh
+fi
+
+if [ -d .git ]
+then
+ mkdir m4 > /dev/zero 2>&1
+ perl config/version.pl || die "Failed to run config/version.pl"
+fi
+
+# Try to detect the supported binaries if the user didn't
+# override that by pushing the environment variable
+if test x$ACLOCAL = x; then
+ ACLOCAL=`locate_binary aclocal-1.11 aclocal-1.10 aclocal`
+ if test x$ACLOCAL = x; then
+ die "Did not find a supported aclocal"
+ fi
+fi
+
+if test x$AUTOMAKE = x; then
+ AUTOMAKE=`locate_binary automake-1.11 automake-1.10 automake`
+ if test x$AUTOMAKE = x; then
+ die "Did not find a supported automake"
+ fi
+fi
+
+if test x$AUTOCONF = x; then
+ AUTOCONF=`locate_binary autoconf`
+ if test x$AUTOCONF = x; then
+ die "Did not find a supported autoconf"
+ fi
+fi
+
+run $ACLOCAL $ACLOCAL_FLAGS || die "Can't execute aclocal"
+run $AUTOMAKE $AUTOMAKE_FLAGS || die "Can't execute automake"
+run $AUTOCONF $AUTOCONF_FLAGS || die "Can't execute autoconf"
+
+if test -f config/post_hook.sh
+then
+ . config/post_hook.sh
+fi
+
+echo "---"
+echo "Configured with the following tools:"
+echo " * `$ACLOCAL --version | head -1`"
+echo " * `$AUTOMAKE --version | head -1`"
+echo " * `$AUTOCONF --version | head -1`"
+echo "---"
View
52 config/version.pl
@@ -0,0 +1,52 @@
+#!/usr/bin/perl
+# If you think this is stupid/overkill, blame dormando
+
+use warnings;
+use strict;
+
+my $version = `git describe`;
+chomp $version;
+# Test the various versions.
+#my $version = 'foob';
+#my $version = '1.4.2-30-gf966dba';
+#my $version = '1.4.3-rc1';
+#my $version = '1.4.3';
+unless ($version =~ m/^\d+\.\d+\.\d+/) {
+ write_file('m4/version.m4', "m4_define([VERSION_NUMBER], [UNKNOWN])\n");
+ exit;
+}
+
+$version =~ s/-/_/g;
+write_file('m4/version.m4', "m4_define([VERSION_NUMBER], [$version])\n");
+my ($VERSION, $FULLVERSION, $RELEASE);
+
+if ($version =~ m/^(\d+\.\d+\.\d+)_rc(\d+)$/) {
+ $VERSION = $1;
+ $FULLVERSION = $version;
+ $RELEASE = '0.1.rc' . $2;
+} elsif ($version =~ m/^(\d+\.\d+\.\d+)_(.+)$/) {
+ $VERSION = $1;
+ $FULLVERSION = $version;
+ $RELEASE = '1.' . $2;
+} elsif ($version =~ m/^(\d+\.\d+\.\d+)$/) {
+ $VERSION = $1;
+ $FULLVERSION = $version;
+ $RELEASE = '1';
+}
+
+sub write_file {
+ my $file = shift;
+ my $data = shift;
+ open(my $fh, "> $file") or die "Can't open $file: $!";
+ print $fh $data;
+ close($fh);
+}
+
+sub read_file {
+ my $file = shift;
+ local $/ = undef;
+ open(my $fh, "< $file") or die "Can't open $file: $!";
+ my $data = <$fh>;
+ close($fh);
+ return $data;
+}
View
12 configure.ac
@@ -0,0 +1,12 @@
+# workload-generator
+# Copyright (C) 2011 Couchbase, INC
+# All rights reserved.
+#
+AC_PREREQ(2.59)
+m4_include([m4/version.m4])
+AC_INIT(workload-generator, VERSION_NUMBER, bin@couchbase.com)
+AC_CONFIG_AUX_DIR(config)
+AM_INIT_AUTOMAKE
+AC_CONFIG_FILES(Makefile wrapper/wrapper)
+
+AC_OUTPUT
View
198 workload-generator
@@ -0,0 +1,198 @@
+#!/usr/bin/python
+#
+# Copyright 2011, Couchbase, Inc.
+# All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import random
+from threading import Thread
+import uuid
+import time
+from couchbase.exception import MemcachedTimeoutException
+from couchbase.couchbaseclient import VBucketAwareCouchbaseClient
+from optparse import OptionParser
+from couchbase.util import ProgressBar, StringUtil
+import sys
+
+class SharedProgressBar(object):
+ def __init__(self, number_of_items):
+ self.bar = ProgressBar(0, number_of_items, 77)
+ self.number_of_items = number_of_items
+ self.counter = 0
+ self.old_bar_string = ""
+
+ def update(self):
+ self.counter += 1
+ if self.old_bar_string != str(self.bar):
+ sys.stdout.write(str(self.bar) + '\r')
+ sys.stdout.flush()
+ self.old_bar_string = str(self.bar)
+ self.bar.updateAmount(self.counter)
+
+ def flush(self):
+ self.bar.updateAmount(self.number_of_items)
+ sys.stdout.write(str(self.bar) + '\r')
+ sys.stdout.flush()
+
+class SmartLoader(object):
+ def __init__(self, options, server, sharedProgressBar, thread_id):
+ self._options = options
+ self._server = server
+ self._thread = None
+ self.shut_down = False
+ self._stats = {"total_time": 0, "max": 0, "min": 1 * 1000 * 1000, "samples": 0, "timeouts": 0}
+ self._bar = sharedProgressBar
+ self._thread_id = thread_id
+
+ def start(self):
+ self._thread = Thread(target=self._run)
+ self._thread.start()
+
+ def _run(self):
+ v = None
+ try:
+ options = self._options
+ v = VBucketAwareCouchbaseClient(self._server, options.bucket, options.password, options.verbose)
+ number_of_items = (int(options.items) / int(options.num_of_threads))
+ value = StringUtil.create_value("*", int(options.value_size))
+ for i in range(0, number_of_items):
+ if self.shut_down:
+ break
+ key = "{0}-{1}".format(options.key_prefix, str(uuid.uuid4())[:5])
+ if options.load_json:
+ document = "\"name\":\"pymc-{0}\"".format(key, key)
+ document = document + ",\"age\":{0}".format(random.randint(0, 1000))
+ document = document + ",\"index\":{0}".format(i)
+ document = "{" + document + "}"
+ try:
+ self._profile_before()
+ v.set(key, 0, 0, document)
+ self._profile_after()
+ except MemcachedTimeoutException:
+ self._stats["timeouts"] += 1
+ else:
+ try:
+ self._profile_before()
+ v.set(key, 0, 0, value)
+ self._profile_after()
+ except MemcachedTimeoutException:
+ self._stats["timeouts"] += 1
+ self._bar.update()
+ v.done()
+ v = None
+ except:
+ if v:
+ v.done()
+
+ def print_stats(self):
+ msg = "Thread {0} - average set time : {1} seconds , min : {2} seconds , max : {3} seconds , operation timeouts {4}"
+ if self._stats["samples"]:
+ print msg.format(self._thread_id, self._stats["total_time"] / self._stats["samples"],
+ self._stats["min"], self._stats["max"], self._stats["timeouts"])
+
+ def wait(self, block=False):
+ if block:
+ self._thread.join()
+ else:
+ return not self._thread.is_alive()
+
+
+ def stop(self):
+ self.shut_down = True
+ if v:
+ v.done()
+
+ def _profile_before(self):
+ self.start = time.time()
+
+ def _profile_after(self):
+ self.end = time.time()
+ diff = self.end - self.start
+ self._stats["samples"] += 1
+ self._stats["total_time"] = self._stats["total_time"] + diff
+ if self._stats["min"] > diff:
+ self._stats["min"] = diff
+ if self._stats["max"] < diff:
+ self._stats["max"] = diff
+
+if __name__ == "__main__":
+ parser = OptionParser()
+ parser.add_option("-n", "--node", dest="node",
+ help="node's ns_server ip:port", metavar="192.168.1.1:8091")
+ parser.add_option("-u", "--user", dest="username",
+ help="node username", metavar="Administrator", default="Administrator")
+ parser.add_option("-p", "--password", dest="password",
+ help="node password", metavar="asdasd")
+ parser.add_option("-b", "--bucket", dest="bucket", help="which bucket to insert data",
+ default="default", metavar="default")
+ parser.add_option("-j", "--json", dest="load_json", help="insert json data",
+ default=False, metavar="True")
+
+ parser.add_option("-v", "--verbose", dest="verbose", help="run in verbose mode",
+ default=False, metavar="False")
+
+ parser.add_option("-i", "--items", dest="items", help="number of items to be inserted",
+ default=100, metavar="100")
+
+ parser.add_option("--size", dest="value_size", help="value size,default is 256 byte",
+ default=512, metavar="100")
+
+ parser.add_option("--threads", dest="num_of_threads", help="number of threads to run load",
+ default=1, metavar="100")
+
+ parser.add_option("--prefix", dest="key_prefix",
+ help="prefix to use for memcached keys and json _ids,default is uuid string",
+ default=str(uuid.uuid4())[:5], metavar="pymc")
+
+ options, args = parser.parse_args()
+
+ node = options.node
+
+ if not node:
+ parser.print_help()
+ exit()
+ #if port is not given use :8091
+ if node.find(":") == -1:
+ hostname = node
+ port = 8091
+ else:
+ hostname = node[:node.find(":")]
+ port = node[node.find(":") + 1:]
+ server = "http://{0}:{1}/pools/default".format(hostname, port)
+ v = None
+ workers = []
+ try:
+ no_threads = options.num_of_threads
+ number_of_items = int(options.items)
+ sharedProgressBar = SharedProgressBar(number_of_items)
+ for i in range(0, int(no_threads)):
+ worker = SmartLoader(options, server, sharedProgressBar, i)
+ worker.start()
+ workers.append(worker)
+ while True:
+ all_finished = True
+ for worker in workers:
+ all_finished &= worker.wait()
+ if all_finished:
+ break
+ else:
+ time.sleep(0.5)
+ sharedProgressBar.flush()
+ for worker in workers:
+ worker.print_stats()
+ except :
+ print ""
+ for worker in workers:
+ worker.stop()
View
12 wrapper/wrapper.in
@@ -0,0 +1,12 @@
+#! /bin/sh
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+root=@libdir@/python
+
+if test -z "${PYTHONPATH}"; then
+ PYTHONPATH=$root
+else
+ PYTHONPATH=$root:${PYTHONPATH}
+fi
+export PYTHONPATH
+exec $root/`basename $0` "$@"
Please sign in to comment.
Something went wrong with that request. Please try again.