Permalink
Browse files

contine design

  • Loading branch information...
bcui6611 committed Apr 13, 2012
1 parent c3756a8 commit 55113f38e4d28dd3857dd873d8bb9a1f582b0423
Showing with 323 additions and 66 deletions.
  1. +11 −3 analyzer.py
  2. +4 −14 buckets.py
  3. +109 −9 cluster_stats.py
  4. +2 −2 dbaccessor.py
  5. +75 −18 diskqueue_stats.py
  6. +27 −20 healthChecker.py
  7. +36 −0 stats_buffer.py
  8. +59 −0 util.py
View
@@ -8,17 +8,25 @@ def __init__(self):
def run_analysis(self):
self.accessor.connect_db()
- self.accessor.browse_db()
+ #self.accessor.browse_db()
- #print cluster_stats.ClusterCapsule
for pill in cluster_stats.ClusterCapsule:
#print pill['name']
for counter in pill['ingredients']:
if counter['type'] == 'SQL':
result = self.accessor.execute(counter['code'])
print counter["description"], ":", result[0]
elif counter['type'] == 'python':
- result = eval("cluster_stats.{0}().run(self.accessor)".format(counter['code']))
+ result = eval("cluster_stats.{0}().run(counter)".format(counter['code']))
+ print counter["description"], ": ", result
+
+ for pill in diskqueue_stats.DiskQueueCapsule:
+ for counter in pill['ingredients']:
+ if counter['type'] == 'SQL':
+ result = self.accessor.execute(counter['code'])
+ print counter["description"], ":", result[0]
+ elif counter['type'] == 'python':
+ result = eval("diskqueue_stats.{0}().run(counter)".format(counter['code']))
print counter["description"], ": ", result
self.accessor.close()
View
@@ -8,25 +8,14 @@
rest_cmds = {
'bucket-list': '/pools/default/buckets',
'bucket-stats': '/pools/default/buckets/{0}/stats?zoom=hour',
- 'bucket-node-stats': '/pools/default/buckets/{0}/stats/{1}?zoom=day'
+ 'bucket-node-stats': '/pools/default/buckets/{0}/stats/{1}?zoom={2}'
}
methods = {
'bucket-list': 'GET',
'bucket-stats': 'GET',
'bucket-node-stats': 'GET',
}
-stats = [
- 'cmd_get',
- 'cmd_set',
- 'delete_hits',
- 'ep_tap_total_total_backlog_size',
- 'curr_items',
- 'curr_items_tot',
- 'disk_write_queue',
- 'curr_connections',
- ]
-
class Buckets:
def __init__(self):
self.debug = False
@@ -96,10 +85,11 @@ def runCmd(self, cmd, server, port,
return rest.getJson(data)
class BucketNodeStats:
- def __init__(self, bucket_name, stat_name):
+ def __init__(self, bucket_name, stat_name, scale):
self.debug = False
- self.rest_cmd = rest_cmds['bucket-node-stats'].format(bucket_name, stat_name)
+ self.rest_cmd = rest_cmds['bucket-node-stats'].format(bucket_name, stat_name, scale)
self.method = 'GET'
+ #print self.rest_cmd
def runCmd(self, cmd, server, port,
user, password, opts):
View
@@ -1,4 +1,6 @@
import dbaccessor
+import stats_buffer
+import util
class DGMRatio:
def run(self, accessor):
@@ -21,14 +23,70 @@ def run(self, accessor):
return ratio
class OpsRatio:
+ def run(self, accessor):
+ ops_avg = {
+ "cmd_get": [],
+ "cmd_set": [],
+ "delete_hits" : [],
+ }
+ for bucket, stats_info in stats_buffer.buckets.iteritems():
+ for counter in accessor["counter"]:
+ values = stats_info[accessor["scale"]][counter]
+ nodeStats = values["nodeStats"]
+ samplesCount = values["samplesCount"]
+ for node, vals in nodeStats.iteritems():
+ avg = sum(vals) / samplesCount
+ ops_avg[counter].append((bucket, node, avg))
+ result = []
+ for read, write, delete in zip(ops_avg['cmd_get'], ops_avg['cmd_set'], ops_avg['delete_hits']):
+
+ total = read[2] + write[2] + delete[2]
+ if total == 0:
+ result.append((read[0], read[1], "0:0:0"))
+ else:
+ read_ratio = read[2] *100 / total
+ write_ratio = write[2] * 100 / total
+ del_ratio = delete[2] * 100 /total
+ result.append((read[0], read[1], "{0}:{1}:{2}".format(read_ratio, write_ratio, del_ratio)))
+
+ return result
+
+class CacheMissRatio:
+ def run(self, accessor):
+ trend = []
+ for bucket, stats_info in stats_buffer.buckets.iteritems():
+ values = stats_info[accessor["scale"]][accessor["counter"]]
+ timestamps = values["timestamp"]
+ timestamps = [x - timestamps[0] for x in timestamps]
+ nodeStats = values["nodeStats"]
+ samplesCount = values["samplesCount"]
+ for node, vals in nodeStats.iteritems():
+ a, b = util.linreg(timestamps, vals)
+ trend.append((bucket, node, a, b))
+ return trend
+
+class ItemGrowth:
+ def run(self, accessor):
+ trend = []
+ for bucket, stats_info in stats_buffer.buckets.iteritems():
+ values = stats_info[accessor["scale"]][accessor["counter"]]
+ timestamps = values["timestamp"]
+ timestamps = [x - timestamps[0] for x in timestamps]
+ nodeStats = values["nodeStats"]
+ samplesCount = values["samplesCount"]
+ for node, vals in nodeStats.iteritems():
+ a, b = util.linreg(timestamps, vals)
+ avg = sum(vals) / samplesCount
+ trend.append((bucket, node, a, b, avg))
+ return trend
+
+class NumVbuckt:
def run(self, accessor):
- ops = accessor.execute("SELECT sum(getOps), sum(setOps), sum(delOps) FROM BucketOps")
- total = accessor.execute("SELECT count(*) FROM BucketOps")
- get_avg = ops[0] / total[0]
- set_avg = ops[1] / total[0]
- del_avg = ops[2] / total[0]
- total_avg = get_avg + set_avg + del_avg
- return (get_avg / total_avg * 100, set_avg / total_avg * 100, del_avg / total_avg * 100)
+ trend = []
+ for bucket, stats_info in stats_buffer.buckets_summary.iteritems():
+ total, values = stats_buffer.retrieveSummaryStats(bucket, accessor["counter"])
+ trend.append((bucket, values[-1]))
+ return trend
ClusterCapsule = [
{"name" : "Node Status",
@@ -63,7 +121,7 @@ def run(self, accessor):
"ingredients" : [
{
"description" : "DGM Ratio",
- "type" : "python",
+ "type" : "SQL",
"code" : "DGMRatio"
},
]
@@ -82,10 +140,52 @@ def run(self, accessor):
{
"description" : "Read/Write/Delete ops ratio",
"type" : "python",
+ "scale" : "minute",
+ "counter" : ["cmd_get", "cmd_set", "delete_hits"],
"code" : "OpsRatio"
},
]
- }
+ },
+ {"name" : "Cache Miss Ratio",
+ "ingredients" : [
+ {
+ "description" : "Cache miss ratio",
+ "counter" : "ep_cache_miss_rate",
+ "type" : "python",
+ "scale" : "hour",
+ "code" : "CacheMissRatio"
+ },
+ ]
+ },
+ {"name" : "Growth Rate",
+ "ingredients" : [
+ {
+ "description" : "Growth rate for items",
+ "counter" : "curr_items",
+ "type" : "python",
+ "scale" : "day",
+ "code" : "ItemGrowth"
+ },
+ ]
+ },
+ {"name" : "VBucket number",
+ "ingredients" : [
+ {
+ "description" : "Active VBucket number",
+ "counter" : "vb_active_num",
+ "type" : "python",
+ "scale" : "summary",
+ "code" : "NumVbuckt"
+ },
+ {
+ "description" : "Replica VBucket number",
+ "counter" : "vb_replica_num",
+ "type" : "python",
+ "scale" : "summary",
+ "code" : "NumVbuckt"
+ },
+ ]
+ },
]
View
@@ -229,8 +229,8 @@ def process_bucket_stats(self, bucket_id, json):
#print "op/sample", json["op"]["samples"]
#print "op/sample/cmd_get", json["op"]["samples"]["cmd_get"]
samples = json["op"]["samples"]
- #for sample in samples.keys():
- # print sample
+ for sample in samples.keys():
+ print "-->",sample
total_samples = json["op"]["samplesCount"]
get_avg = sum(json["op"]["samples"]["cmd_get"]) / total_samples
set_avg = sum(json["op"]["samples"]["cmd_set"]) / total_samples
View
@@ -1,39 +1,96 @@
import dbaccessor
+import stats_buffer
+import util
+counter_name = 'disk_write_queue'
-class DiskQueue:
+class AvgDiskQueue:
def run(self, accessor):
- queue_size = accessor.execute("SELECT sum(diskWriteQueue) FROM BucketOps")
- total = accessor.execute("SELECT count(*) FROM BucketOps")
- disk_queue_avg = queue_size[0] / total[0]
- return (disk_queue_avg)
+ disk_queue_avg = []
-class TotalOps:
+ for bucket, stats_info in stats_buffer.buckets.iteritems():
+ #print bucket, stats_info
+ values = stats_info[accessor["scale"]][accessor["counter"]]
+ nodeStats = values["nodeStats"]
+ samplesCount = values["samplesCount"]
+ for node, vals in nodeStats.iteritems():
+ avg = sum(vals) / samplesCount
+ disk_queue_avg.append((bucket, node, avg))
+ return disk_queue_avg
+
+ def action(self, values, thresholds):
+ flags = []
+ for bucket, node, avg in values:
+ if avg < thresholds["low"]:
+ flags.append((bucket, node, "green"))
+ elif avg >= thresholds["low"] and avg < thresholds["high"]:
+ flags.append((bucket, node, "yellow"))
+ else:
+ flags.append((bucket, node, "red"))
+ return flags
+
+class DiskQueueTrend:
def run(self, accessor):
- ops = accessor.execute("SELECT sum(getOps), sum(setOps), sum(delOps) FROM BucketOps")
- total = accessor.execute("SELECT count(*) FROM BucketOps")
- get_avg = ops[0] / total[0]
- set_avg = ops[1] / total[0]
- del_avg = ops[2] / total[0]
- total_avg = get_avg + set_avg + del_avg
- return (total_avg, get_avg / total_avg * 100, set_avg / total_avg * 100, del_avg / total_avg * 100)
+ trend = []
+ for bucket, stats_info in stats_buffer.buckets.iteritems():
+ values = stats_info[accessor["scale"]][accessor["counter"]]
+ timestamps = values["timestamp"]
+ timestamps = [x - timestamps[0] for x in timestamps]
+ nodeStats = values["nodeStats"]
+ samplesCount = values["samplesCount"]
+ for node, vals in nodeStats.iteritems():
+ a, b = util.linreg(timestamps, vals)
+ trend.append((bucket, node, a, b))
+ return trend
+
+class TapQueueTrend:
+ def run(self, accessor):
+ trend = []
+ for bucket, stats_info in stats_buffer.buckets.iteritems():
+ values = stats_info[accessor["scale"]][accessor["counter"]]
+ timestamps = values["timestamp"]
+ timestamps = [x - timestamps[0] for x in timestamps]
+ nodeStats = values["nodeStats"]
+ samplesCount = values["samplesCount"]
+ for node, vals in nodeStats.iteritems():
+ a, b = util.linreg(timestamps, vals)
+ trend.append((bucket, node, a, b))
+ return trend
DiskQueueCapsule = [
{"name" : "Disk Queue Diagnosis",
"ingredients" : [
{
"description" : "Avg Disk queue length",
+ "counter" : "disk_write_queue",
+ "pernode" : True,
+ "scale" : "minute",
"type" : "python",
- "code" : "DiskQueue",
+ "code" : "AvgDiskQueue",
"threshold" : {
"low" : 50000000,
"high" : 1000000000
- }
+ },
},
{
- "description" : "Total OPS",
+ "description" : "Disk queue trend",
+ "counter" : "disk_write_queue",
+ "pernode" : True,
+ "scale" : "hour",
+ "type" : "python",
+ "code" : "DiskQueueTrend",
+ },
+ ]
+ },
+ {"name" : "Replication trend",
+ "ingredients" : [
+ {
+ "description" : "Replication Trend",
+ "counter" : "ep_tap_total_total_backlog_size",
+ "pernode" : True,
+ "scale" : "hour",
"type" : "python",
- "code" : "TotalOps"
+ "code" : "TapQueueTrend",
}
]
- }
+ },
]
Oops, something went wrong.

0 comments on commit 55113f3

Please sign in to comment.