Skip to content

Commit

Permalink
Generate relative feature complete report
Browse files Browse the repository at this point in the history
  • Loading branch information
@bcui6611
bcui6611 committed Jun 12, 2012
1 parent e9073c3 commit 63f75e9
Show file tree
Hide file tree
Showing 29 changed files with 2,133 additions and 1,153 deletions.
4 changes: 2 additions & 2 deletions Makefile.am
View file
Expand Up @@ -4,9 +4,9 @@ default:

pythonlibdir=$(libdir)/python

pythonlib_SCRIPTS= cbworkloadgen
pythonlib_SCRIPTS= healthChecker

PYTHON_TOOLS= wrapper/cbworkloadgen
PYTHON_TOOLS= wrapper/healthChecker

${PYTHON_TOOLS}: wrapper/wrapper
cp $< $@
Expand Down
Empty file removed README
View file
Empty file.
148 changes: 83 additions & 65 deletions analyzer.py
View file
@@ -1,12 +1,11 @@
import sys
import datetime
import dbaccessor
import util
import logging

import util_cli as util
import cluster_stats
import bucket_stats
import diskqueue_stats
import node_stats

import stats_buffer

from Cheetah.Template import Template
Expand All @@ -29,68 +28,45 @@
cluster_symptoms = {}
bucket_symptoms = {}
bucket_node_symptoms = {}
bucket_node_status = {}
node_symptoms = {}
indicator_error = {}
indicator_warn = {}
node_disparate = {}

def format_output(counter, result):
if len(result) == 1:
if counter.has_key("unit") and counter["unit"] == "GB":
return util.pretty_float(result[0])
else:
return result[0]
else:
return result

class StatsAnalyzer:
def __init__(self):
self.accessor = dbaccessor.DbAccesor()
def __init__(self, log):
self.log = log

def run_analysis(self):
self.accessor.connect_db()
self.accessor.browse_db()

for bucket in stats_buffer.buckets.iterkeys():
bucket_list.append(bucket)
bucket_symptoms[bucket] = []
bucket_node_symptoms[bucket] = {}
bucket_node_status[bucket] = {}

for capsule, package_name in capsules:
for pill in capsule:
#print pill['name']
self.log.debug(pill['name'])
for counter in pill['ingredients']:
if counter['type'] == 'SQL':
result = eval("{0}.{1}().run(self.accessor, \"{2}\")".format(package_name, counter['code'], counter['stmt']))
elif counter['type'] == 'pythonSQL':
result = eval("{0}.{1}().run(self.accessor)".format(package_name, counter['code']))
elif counter['type'] == 'python':
result = eval("{0}.{1}().run(counter)".format(package_name, counter['code']))

#if counter.has_key("unit") and counter["unit"] == "GB":
# util.pretty_print({counter["description"] : result})
#else:
# util.pretty_print({counter["description"] : result})
result = eval("{0}.{1}().run(counter)".format(package_name, counter['code']))

#print counter
self.log.debug(counter)
if pill.has_key("clusterwise") and pill["clusterwise"] :
if isinstance(result, dict):
if result.has_key("cluster"):
if counter.has_key("unit") and counter["unit"] == "GB":
cluster_symptoms[counter["name"]] = {"description" : counter["description"], "value": util.humanize_bytes(result["cluster"])}
else:
cluster_symptoms[counter["name"]] = {"description" : counter["description"], "value":result["cluster"]}
cluster_symptoms[counter["name"]] = {"description" : counter["description"], "value":result["cluster"]}
else:
cluster_symptoms[counter["name"]] = {"description" : counter["description"], "value":result}
else:
cluster_symptoms[counter["name"]] = {"description" : counter["description"], "value":result}
if pill.has_key("perBucket") and pill["perBucket"] :
#bucket_symptoms[counter["name"]] = {"description" : counter["description"], "value":result}
for bucket, values in result.iteritems():
if bucket == "cluster":
continue
for val in values:
if val[0] == "variance":
if val[0] == "variance" or val[0] == "error":
continue
elif val[0] == "total":
bucket_symptoms[bucket].append({"description" : counter["description"], "value" : values[-1][1]})
Expand All @@ -104,14 +80,49 @@ def run_analysis(self):
if pill.has_key("nodewise") and pill["nodewise"]:
node_list[counter["name"]] = {"description" : counter["description"], "value":result}

if pill.has_key("indicator") and pill["indicator"] :
if pill.has_key("indicator"):
if len(result) > 0:
for bucket,values in result.iteritems():
if values.has_key("error"):
indicator_error[counter["name"]] = {"description" : counter["description"], "bucket": bucket, "value":values["error"]}
if values.has_key("warn"):
indicator_warn[counter["name"]] = {"description" : counter["description"], "bucket": bucket, "value":values["warn"]}

if type(values) is dict:
if values.has_key("error"):
indicator_error[counter["name"]] = {"description" : counter["description"],
"bucket": bucket,
"value":values["error"],
"cause" : pill["indicator"]["cause"],
"impact" : pill["indicator"]["impact"],
"action" : pill["indicator"]["action"],
}
for val in values["error"]:
bucket_node_status[bucket][val["node"]] = "error"

if values.has_key("warn"):
indicator_warn[counter["name"]] = {"description" : counter["description"],
"bucket": bucket,
"value":values["warn"],
"cause" : pill["indicator"]["cause"],
"impact" : pill["indicator"]["impact"],
"action" : pill["indicator"]["action"],
}
elif type(values) is list:
for val in values:
if val[0] == "error":
indicator_error[counter["name"]] = {"description" : counter["description"],
"bucket": bucket,
"value":val[1],
"cause" : pill["indicator"]["cause"],
"impact" : pill["indicator"]["impact"],
"action" : pill["indicator"]["action"],
}
for val in values["error"]:
bucket_node_status[bucket][val["node"]] = "error"
elif val[0] == "warn":
indicator_warn[counter["name"]] = {"description" : counter["description"],
"bucket": bucket,
"value":val[1],
"cause" : pill["indicator"]["cause"],
"impact" : pill["indicator"]["impact"],
"action" : pill["indicator"]["action"],
}
if pill.has_key("nodeDisparate") and pill["nodeDisparate"] :
for bucket,values in result.iteritems():
if bucket == "cluster":
Expand All @@ -121,42 +132,49 @@ def run_analysis(self):
continue;
if val[0] == "variance" and val[1] != 0:
node_disparate[counter["name"]] = {"description" : counter["description"], "bucket": bucket, "value":values}

self.accessor.close()
self.accessor.remove_db()

def run_report(self):

if len(indicator_error) > 0:
globals["cluster_health"] = "error"
elif len(indicator_warn) > 0:
globals["cluster_health"] = "warning"

def run_report(self, txtfile, htmlfile, verbose):

dict = {
"globals" : globals,
"cluster_symptoms" : cluster_symptoms,
"bucket_symptoms" : bucket_symptoms,
"bucket_node_symptoms" : bucket_node_symptoms,
"bucket_node_status" : bucket_node_status,
"node_symptoms" : node_symptoms,
"node_list" : node_list,
"bucket_list" : bucket_list,
"indicator_warn" : indicator_warn,
"indicator_error" : indicator_error,
"verbose" : verbose,
}

debug = True
if debug:
print "Nodelist Overview"
util.pretty_print(node_list)
f = open(txtfile, 'w')
report = {}
report["Report Time"] = globals["report_time"].strftime("%Y-%m-%d %H:%M:%S")

report["Nodelist Overview"] = node_list

print "Cluster Overview"
util.pretty_print(cluster_symptoms)

print "Bucket Metrics"
util.pretty_print(bucket_symptoms)

print "Bucket Node Metrics"
util.pretty_print(bucket_node_symptoms)
report["Cluster Overview"] = cluster_symptoms

report["Bucket Metrics"] = bucket_symptoms

report["Bucket Node Metrics"] = bucket_node_symptoms

print "Key indicators"
util.pretty_print(indicator_error)
util.pretty_print(indicator_warn)
report["Key indicators"] = (indicator_error, indicator_warn)

print "Node disparate"
util.pretty_print(node_disparate)
#print Template(file="report-htm.tmpl", searchList=[dict])
report["Node disparate"] = node_disparate

print >> f, util.pretty_print(report)
f.close()

f = open(htmlfile, 'w')
print >> f, Template(file="report-htm.tmpl", searchList=[dict])
f.close()

sys.stderr.write("\nThis run finishes successfully. Please find output result at " + htmlfile)
70 changes: 67 additions & 3 deletions buckets.py
View file
Expand Up @@ -7,11 +7,21 @@

rest_cmds = {
'bucket-list': '/pools/default/buckets',
'bucket-flush': '/pools/default/buckets/',
'bucket-delete': '/pools/default/buckets/',
'bucket-create': '/pools/default/buckets/',
'bucket-edit': '/pools/default/buckets/',
'bucket-get': '/pools/default/buckets',
'bucket-stats': '/pools/default/buckets/{0}/stats?zoom=hour',
'bucket-node-stats': '/pools/default/buckets/{0}/stats/{1}?zoom={2}'
}
methods = {
'bucket-list': 'GET',
'bucket-delete': 'DELETE',
'bucket-create': 'POST',
'bucket-edit': 'POST',
'bucket-flush': 'POST',
'bucket-get': 'GET',
'bucket-stats': 'GET',
'bucket-node-stats': 'GET',
}
Expand Down Expand Up @@ -58,13 +68,68 @@ def runCmd(self, cmd, server, port,

# get the parameters straight

if cmd in ('bucket-create', 'bucket-edit'):
if bucketname:
rest.setParam('name', bucketname)
if bucketname == "default":
if bucketport and bucketport != "11211":
usage("default bucket must be on port 11211.")
if bucketpassword:
usage("default bucket should only have empty password.")
authtype = 'sasl'
else:
if bucketport == "11211":
authtype = 'sasl'
else:
authtype = 'none'
if bucketpassword:
usage("a sasl bucket is supported only on port 11211.")
if buckettype:
rest.setParam('bucketType', buckettype)
if authtype:
rest.setParam('authType', authtype)
if bucketport:
rest.setParam('proxyPort', bucketport)
if bucketpassword:
rest.setParam('saslPassword', bucketpassword)
if bucketramsize:
rest.setParam('ramQuotaMB', bucketramsize)
if bucketreplication:
rest.setParam('replicaNumber', bucketreplication)
if cmd in ('bucket-delete', 'bucket-flush', 'bucket-edit'):
self.rest_cmd = self.rest_cmd + bucketname
if cmd == 'bucket-flush':
self.rest_cmd = self.rest_cmd + '/controller/doFlush'

opts = {}
opts['error_msg'] = "unable to %s" % cmd
opts['error_msg'] = "unable to %s; please check your username (-u) and password (-p);" % cmd
opts['success_msg'] = "%s" % cmd
data = rest.restCmd(methods[cmd], self.rest_cmd,
self.user, self.password, opts)

return rest.getJson(data)
if cmd in("bucket-get", "bucket-stats", "bucket-node-stats"):
return rest.getJson(data)
elif cmd == "bucket-list":
if output == 'json':
print data
else:
json = rest.getJson(data)
for bucket in json:
print '%s' % bucket['name']
print ' bucketType: %s' % bucket['bucketType']
print ' authType: %s' % bucket['authType']
if bucket['authType'] == "sasl":
print ' saslPassword: %s' % bucket['saslPassword']
else:
print ' proxyPort: %s' % bucket['proxyPort']
print ' numReplicas: %s' % bucket['replicaNumber']
print ' ramQuota: %s' % bucket['quota']['ram']
print ' ramUsed: %s' % bucket['basicStats']['memUsed']
else:
if output == 'json':
print rest.jsonMessage(data)
else:
print data

class BucketStats:
def __init__(self, bucket_name):
Expand Down Expand Up @@ -102,4 +167,3 @@ def runCmd(self, cmd, server, port,
data = rest.restCmd(methods[cmd], self.rest_cmd,
user, password, opts)
return rest.getJson(data)

0 comments on commit 63f75e9

Please sign in to comment.