Permalink
Browse files

Add html template support

  • Loading branch information...
1 parent 10a0fca commit 96cd926dedc09d92f9f8f64b8913f661e96a545e @bcui6611 bcui6611 committed May 21, 2012
Showing with 574 additions and 25 deletions.
  1. +22 −0 Makefile.am
  2. +22 −6 analyzer.py
  3. +13 −6 bucket_stats.py
  4. +50 −6 cluster_stats.py
  5. +81 −0 config/autorun.sh
  6. +52 −0 config/version.pl
  7. +12 −0 configure.ac
  8. +56 −3 diskqueue_stats.py
  9. +2 −2 healthChecker.py
  10. +237 −0 htmlreport.tmpl
  11. +8 −2 stats_buffer.py
  12. +19 −0 util.py
View
@@ -0,0 +1,22 @@
+COUCHBASE_PYTHON_LIB = ../couchbase-python-client/couchbase
+
+default:
+
+pythonlibdir=$(libdir)/python
+
+pythonlib_SCRIPTS= cbworkloadgen
+
+PYTHON_TOOLS= wrapper/cbworkloadgen
+
+${PYTHON_TOOLS}: wrapper/wrapper
+ cp $< $@
+
+CLEANFILES = ${PYTHON_TOOLS}
+bin_SCRIPTS = ${PYTHON_TOOLS}
+
+EXTRA_DIST = $(pythonlib_SCRIPTS)
+
+install-data-hook:
+ cp -rf $(COUCHBASE_PYTHON_LIB) $(pythonlibdir)
+
+
View
@@ -32,6 +32,7 @@
node_symptoms = {}
indicator_error = {}
indicator_warn = {}
+node_disparate = {}
def format_output(counter, result):
if len(result) == 1:
@@ -88,12 +89,15 @@ def run_analysis(self):
for bucket, values in result.iteritems():
if bucket == "cluster":
continue
- if values[-1][0] == "total":
- bucket_symptoms[bucket].append({"description" : counter["description"], "value" : values[-1][1]})
- for val in values[:-1]:
- if bucket_node_symptoms[bucket].has_key(val[0]) == False:
- bucket_node_symptoms[bucket][val[0]] = []
- bucket_node_symptoms[bucket][val[0]].append({"description" : counter["description"], "value" : val[1]})
+ for val in values:
+ if val[0] == "variance":
+ continue
+ elif val[0] == "total":
+ bucket_symptoms[bucket].append({"description" : counter["description"], "value" : values[-1][1]})
+ else:
+ if bucket_node_symptoms[bucket].has_key(val[0]) == False:
+ bucket_node_symptoms[bucket][val[0]] = []
+ bucket_node_symptoms[bucket][val[0]].append({"description" : counter["description"], "value" : val[1]})
if pill.has_key("perNode") and pill["perNode"] :
node_symptoms[counter["name"]] = {"description" : counter["description"], "value":result}
@@ -108,6 +112,16 @@ def run_analysis(self):
if values.has_key("warn"):
indicator_warn[counter["name"]] = {"description" : counter["description"], "bucket": bucket, "value":values["warn"]}
+ if pill.has_key("nodeDisparate") and pill["nodeDisparate"] :
+ for bucket,values in result.iteritems():
+ if bucket == "cluster":
+ continue
+ for val in values:
+ if val[0] == "total":
+ continue;
+ if val[0] == "variance" and val[1] != 0:
+ node_disparate[counter["name"]] = {"description" : counter["description"], "bucket": bucket, "value":values}
+
self.accessor.close()
self.accessor.remove_db()
@@ -143,4 +157,6 @@ def run_report(self):
util.pretty_print(indicator_error)
util.pretty_print(indicator_warn)
+ print "Node disparate"
+ util.pretty_print(node_disparate)
#print Template(file="report-htm.tmpl", searchList=[dict])
View
@@ -98,11 +98,18 @@ def run(self, accessor):
class NumVbuckt:
def run(self, accessor):
- trend = []
- for bucket, stats_info in stats_buffer.buckets_summary.iteritems():
+ result = {}
+ for bucket, stats_info in stats_buffer.buckets.iteritems():
+ num_error = []
total, values = stats_buffer.retrieveSummaryStats(bucket, accessor["counter"])
- trend.append((bucket, values[-1]))
- return trend
+ values = stats_info[accessor["scale"]][accessor["counter"]]
+ nodeStats = values["nodeStats"]
+ for node, vals in nodeStats.iteritems():
+ if vals[-1] < accessor["threshold"]:
+ num_error.append({"node":node, "value":vals[-1]})
+ if len(num_error) > 0:
+ result[bucket] = {"error" : num_error}
+ return result
BucketCapsule = [
{"name" : "bucketList",
@@ -178,14 +185,14 @@ def run(self, accessor):
"description" : "Active VBucket number",
"counter" : "vb_active_num",
"type" : "python",
- "scale" : "minute",
+ "scale" : "hour",
"code" : "NumVbuckt"
},
{
"description" : "Replica VBucket number",
"counter" : "vb_replica_num",
"type" : "python",
- "scale" : "minute",
+ "scale" : "hour",
"code" : "NumVbuckt"
},
]
View
@@ -103,18 +103,42 @@ def run(self, accessor):
samplesCount = values["samplesCount"]
trend = []
total = 0
+ data = []
for node, vals in nodeStats.iteritems():
a, b = util.linreg(timestamps, vals)
value = a * timestamps[-1] + b
total += value
trend.append((node, util.pretty_float(value)))
+ data.append(value)
total /= len(nodeStats)
trend.append(("total", util.pretty_float(total)))
+ trend.append(("variance", util.two_pass_variance(data)))
cluster += total
result[bucket] = trend
result["cluster"] = util.pretty_float(cluster / len(stats_buffer.buckets))
return result
+class MemUsed:
+ def run(self, accessor):
+ result = {}
+ cluster = 0
+ for bucket, stats_info in stats_buffer.buckets.iteritems():
+ values = stats_info[accessor["scale"]][accessor["counter"]]
+ timestamps = values["timestamp"]
+ timestamps = [x - timestamps[0] for x in timestamps]
+ nodeStats = values["nodeStats"]
+ samplesCount = values["samplesCount"]
+ trend = []
+ total = 0
+ data = []
+ for node, vals in nodeStats.iteritems():
+ avg = sum(vals) / samplesCount
+ trend.append((node, util.pretty_float(avg)))
+ data.append(avg)
+ trend.append(("variance", util.two_pass_variance(data)))
+ result[bucket] = trend
+ return result
+
class ItemGrowth:
def run(self, accessor):
result = {}
@@ -144,10 +168,15 @@ def run(self, accessor):
class NumVbuckt:
def run(self, accessor):
result = {}
- for bucket, stats_info in stats_buffer.buckets_summary.iteritems():
- total, values = stats_buffer.retrieveSummaryStats(bucket, accessor["counter"])
- if values[-1] < accessor["threshold"]:
- result[bucket] = values[-1]
+ for bucket, stats_info in stats_buffer.buckets.iteritems():
+ num_error = []
+ values = stats_info[accessor["scale"]][accessor["counter"]]
+ nodeStats = values["nodeStats"]
+ for node, vals in nodeStats.iteritems():
+ if vals[-1] < accessor["threshold"]:
+ num_error.append({"node":node, "value":vals[-1]})
+ if len(num_error) > 0:
+ result[bucket] = {"error" : num_error}
return result
ClusterCapsule = [
@@ -198,6 +227,7 @@ def run(self, accessor):
"perNode" : True,
"perBucket" : True,
"indicator" : False,
+ "nodeDisparate" : True,
},
{"name" : "DGM",
"ingredients" : [
@@ -270,7 +300,7 @@ def run(self, accessor):
"description" : "Active VBucket number is less than expected",
"counter" : "vb_active_num",
"type" : "python",
- "scale" : "summary",
+ "scale" : "hour",
"code" : "NumVbuckt",
"threshold" : 1024,
},
@@ -279,13 +309,27 @@ def run(self, accessor):
"description" : "Replica VBucket number is less than expected",
"counter" : "vb_replica_num",
"type" : "python",
- "scale" : "summary",
+ "scale" : "hour",
"code" : "NumVbuckt",
"threshold" : 1024,
},
],
"indicator" : True,
},
+ {"name" : "MemoryUsage",
+ "ingredients" : [
+ {
+ "name" : "memoryUsage",
+ "description" : "Check if memory usage and/or fragmentaion",
+ "type" : "python",
+ "counter" : "mem_used",
+ "scale" : "hour",
+ "code" : "MemUsed",
+ },
+ ],
+ "perNode" : True,
+ "nodeDisparate" : True,
+ },
]
View
@@ -0,0 +1,81 @@
+#!/bin/sh
+
+AUTOMAKE_FLAGS="--add-missing --copy --force --foreign --warning=portability"
+ACLOCAL_FLAGS="-I m4"
+AUTOCONF_CLAGS="--warnings=error --force"
+
+ARGV0=$0
+ARGS="$@"
+
+die() { echo "$@"; exit 1; }
+
+run() {
+ echo "$ARGV0: running \`$@' $ARGS"
+ $@ $ARGS
+}
+
+# Try to locate a program by using which, and verify that the file is an
+# executable
+locate_binary() {
+ for f in $@
+ do
+ file=`which $f 2>/dev/null | grep -v '^no '`
+ if test -n "$file" -a -x "$file"; then
+ echo $file
+ return 0
+ fi
+ done
+
+ echo ""
+ return 1
+}
+
+if test -f config/pre_hook.sh
+then
+ . config/pre_hook.sh
+fi
+
+if [ -d .git ]
+then
+ mkdir m4 > /dev/zero 2>&1
+ perl config/version.pl || die "Failed to run config/version.pl"
+fi
+
+# Try to detect the supported binaries if the user didn't
+# override that by pushing the environment variable
+if test x$ACLOCAL = x; then
+ ACLOCAL=`locate_binary aclocal-1.11 aclocal-1.10 aclocal`
+ if test x$ACLOCAL = x; then
+ die "Did not find a supported aclocal"
+ fi
+fi
+
+if test x$AUTOMAKE = x; then
+ AUTOMAKE=`locate_binary automake-1.11 automake-1.10 automake`
+ if test x$AUTOMAKE = x; then
+ die "Did not find a supported automake"
+ fi
+fi
+
+if test x$AUTOCONF = x; then
+ AUTOCONF=`locate_binary autoconf`
+ if test x$AUTOCONF = x; then
+ die "Did not find a supported autoconf"
+ fi
+fi
+
+run $ACLOCAL $ACLOCAL_FLAGS || die "Can't execute aclocal"
+run $AUTOMAKE $AUTOMAKE_FLAGS || die "Can't execute automake"
+run $AUTOCONF $AUTOCONF_FLAGS || die "Can't execute autoconf"
+
+if test -f config/post_hook.sh
+then
+ . config/post_hook.sh
+fi
+
+echo "---"
+echo "Configured with the following tools:"
+echo " * `$ACLOCAL --version | head -1`"
+echo " * `$AUTOMAKE --version | head -1`"
+echo " * `$AUTOCONF --version | head -1`"
+echo "---"
View
@@ -0,0 +1,52 @@
+#!/usr/bin/perl
+# If you think this is stupid/overkill, blame dormando
+
+use warnings;
+use strict;
+
+my $version = `git describe`;
+chomp $version;
+# Test the various versions.
+#my $version = 'foob';
+#my $version = '1.4.2-30-gf966dba';
+#my $version = '1.4.3-rc1';
+#my $version = '1.4.3';
+unless ($version =~ m/^\d+\.\d+\.\d+/) {
+ write_file('m4/version.m4', "m4_define([VERSION_NUMBER], [UNKNOWN])\n");
+ exit;
+}
+
+$version =~ s/-/_/g;
+write_file('m4/version.m4', "m4_define([VERSION_NUMBER], [$version])\n");
+my ($VERSION, $FULLVERSION, $RELEASE);
+
+if ($version =~ m/^(\d+\.\d+\.\d+)_rc(\d+)$/) {
+ $VERSION = $1;
+ $FULLVERSION = $version;
+ $RELEASE = '0.1.rc' . $2;
+} elsif ($version =~ m/^(\d+\.\d+\.\d+)_(.+)$/) {
+ $VERSION = $1;
+ $FULLVERSION = $version;
+ $RELEASE = '1.' . $2;
+} elsif ($version =~ m/^(\d+\.\d+\.\d+)$/) {
+ $VERSION = $1;
+ $FULLVERSION = $version;
+ $RELEASE = '1';
+}
+
+sub write_file {
+ my $file = shift;
+ my $data = shift;
+ open(my $fh, "> $file") or die "Can't open $file: $!";
+ print $fh $data;
+ close($fh);
+}
+
+sub read_file {
+ my $file = shift;
+ local $/ = undef;
+ open(my $fh, "< $file") or die "Can't open $file: $!";
+ my $data = <$fh>;
+ close($fh);
+ return $data;
+}
View
@@ -0,0 +1,12 @@
+# workload-generator
+# Copyright (C) 2011 Couchbase, INC
+# All rights reserved.
+#
+AC_PREREQ(2.59)
+m4_include([m4/version.m4])
+AC_INIT(workload-generator, VERSION_NUMBER, bin@couchbase.com)
+AC_CONFIG_AUX_DIR(config)
+AM_INIT_AUTOMAKE
+AC_CONFIG_FILES(Makefile wrapper/wrapper)
+
+AC_OUTPUT
Oops, something went wrong.

0 comments on commit 96cd926

Please sign in to comment.