Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100755 234 lines (205 sloc) 10.499 kb
68f5de7 @bcui6611 Add collector module
authored
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3
4 import sys
5 import traceback
6 import copy
7 import logging
8
9 import listservers
10 import buckets
11 #import node
12 import info
13 import util_cli as util
14 import mc_bin_client
15
16 import stats_buffer
17
18 class StatsCollector:
19 def __init__(self, log):
20 self.log = log
21
22 def seg(self, k, v):
23 # Parse ('some_stat_x_y', 'v') into (('some_stat', x, y), v)
24 ka = k.split('_')
25 k = '_'.join(ka[0:-1])
26 kstart, kend = [int(x) for x in ka[-1].split(',')]
27 return ((k, kstart, kend), int(v))
28
29 def retrieve_node_stats(self, nodeInfo, nodeStats):
30 nodeStats['portDirect'] = nodeInfo['ports']['direct']
31 nodeStats['portProxy'] = nodeInfo['ports']['proxy']
32 nodeStats['clusterMembership'] = nodeInfo['clusterMembership']
33 nodeStats['os'] = nodeInfo['os']
34 nodeStats['uptime'] = nodeInfo['uptime']
35 nodeStats['version'] = nodeInfo['version']
36
37 #memory
38 nodeStats['memory'] = {}
39 nodeStats['memory']['allocated'] = nodeInfo['mcdMemoryAllocated']
40 nodeStats['memory']['reserved'] = nodeInfo['mcdMemoryReserved']
41 nodeStats['memory']['free'] = nodeInfo['memoryFree']
42 nodeStats['memory']['quota'] = nodeInfo['memoryQuota']
43 nodeStats['memory']['total'] = nodeInfo['memoryTotal']
44
45 #storageInfo
46 nodeStats['StorageInfo'] = {}
47 if nodeInfo['storageTotals'] is not None:
48
49 #print nodeInfo
50 hdd = nodeInfo['storageTotals']['hdd']
51 if hdd is not None:
52 nodeStats['StorageInfo']['hdd'] = {}
53 nodeStats['StorageInfo']['hdd']['free'] = hdd['free']
54 nodeStats['StorageInfo']['hdd']['quotaTotal'] = hdd['quotaTotal']
55 nodeStats['StorageInfo']['hdd']['total'] = hdd['total']
56 nodeStats['StorageInfo']['hdd']['used'] = hdd['used']
57 nodeStats['StorageInfo']['hdd']['usedByData'] = hdd['usedByData']
58 ram = nodeInfo['storageTotals']['ram']
59 if ram is not None:
60 nodeStats['StorageInfo']['ram'] = {}
61 nodeStats['StorageInfo']['ram']['quotaTotal'] = ram['quotaTotal']
62 nodeStats['StorageInfo']['ram']['total'] = ram['total']
63 nodeStats['StorageInfo']['ram']['used'] = ram['used']
64 nodeStats['StorageInfo']['ram']['usedByData'] = ram['usedByData']
65 if ram.has_key('quotaUsed'):
66 nodeStats['StorageInfo']['ram']['quotaUsed'] = ram['quotaUsed']
67 else:
68 nodeStats['StorageInfo']['ram']['quotaUsed'] = 0
69
70 #system stats
71 nodeStats['systemStats'] = {}
72 nodeStats['systemStats']['cpu_utilization_rate'] = nodeInfo['systemStats']['cpu_utilization_rate']
73 nodeStats['systemStats']['swap_total'] = nodeInfo['systemStats']['swap_total']
74 nodeStats['systemStats']['swap_used'] = nodeInfo['systemStats']['swap_used']
75
76 curr_items = 0
77 curr_items_tot = 0
78 vb_rep_curr_items = 0
79 if nodeInfo['interestingStats'] is not None:
80 if nodeInfo['interestingStats'].has_key('curr_items'):
81 curr_items = nodeInfo['interestingStats']['curr_items']
82 else:
83 curr_items = 0
84 if nodeInfo['interestingStats'].has_key('curr_items_tot'):
85 curr_items_tot = nodeInfo['interestingStats']['curr_items_tot']
86 else:
87 curr_items_tot = 0
88 if nodeInfo['interestingStats'].has_key('vb_replica_curr_items'):
89 vb_rep_curr_items = nodeInfo['interestingStats']['vb_replica_curr_items']
90 else:
91 vb_rep_curr_items = 0
92
93 nodeStats['systemStats']['currentItems'] = curr_items
94 nodeStats['systemStats']['currentItemsTotal'] = curr_items_tot
95 nodeStats['systemStats']['replicaCurrentItems'] = vb_rep_curr_items
96
97 def get_hostlist(self, server, port, user, password, opts):
98 try:
99 opts.append(("-o", "return"))
100 nodes = listservers.ListServers().runCmd('host-list', server, port, user, password, opts)
101
102 for node in nodes:
103 (node_server, node_port) = util.hostport(node['hostname'])
104 node_stats = {"host" : node_server,
105 "port" : node_port,
106 "status" : node['status'],
107 "master" : server}
108 stats_buffer.nodes[node['hostname']] = node_stats
109 if node['status'] == 'healthy':
110 node_info = info.Info().runCmd('get-server-info', node_server, node_port, user, password, opts)
111 self.retrieve_node_stats(node_info, node_stats)
112 else:
113 self.log.error("Unhealthy node: %s:%s" %(node_server, node['status']))
114 return nodes
115 except Exception, err:
116 traceback.print_exc()
117 sys.exit(1)
118
119 def get_bucketlist(self, server, port, user, password, opts):
120 try:
121 bucketlist = buckets.Buckets().runCmd('bucket-get', server, port, user, password, opts)
122 for bucket in bucketlist:
123 bucket_name = bucket['name']
124 self.log.info("bucket: %s" % bucket_name)
125 bucketinfo = {}
126 bucketinfo['name'] = bucket_name
127 bucketinfo['bucketType'] = bucket['bucketType']
128 bucketinfo['authType'] = bucket['authType']
129 bucketinfo['saslPassword'] = bucket['saslPassword']
130 bucketinfo['numReplica'] = bucket['replicaNumber']
131 bucketinfo['ramQuota'] = bucket['quota']['ram']
132 bucketinfo['master'] = server
133
134 bucketStats = bucket['basicStats']
135 bucketinfo['bucketStats'] = {}
136 bucketinfo['bucketStats']['diskUsed'] = bucketStats['diskUsed']
137 bucketinfo['bucketStats']['memUsed'] = bucketStats['memUsed']
138 bucketinfo['bucketStats']['diskFetches'] = bucketStats['diskFetches']
139 bucketinfo['bucketStats']['quotaPercentUsed'] = bucketStats['quotaPercentUsed']
140 bucketinfo['bucketStats']['opsPerSec'] = bucketStats['opsPerSec']
141 bucketinfo['bucketStats']['itemCount'] = bucketStats['itemCount']
142
143 stats_buffer.bucket_info[bucket_name] = bucketinfo
144
145 # get bucket related stats
146 c = buckets.BucketStats(bucket_name)
147 json = c.runCmd('bucket-stats', server, port, user, password, opts)
148 stats_buffer.buckets_summary[bucket_name] = json
149 return bucketlist
150 except Exception, err:
151 traceback.print_exc()
152 sys.exit(1)
153
154 def get_mc_stats_per_node(self, mc, stats):
155 cmd_list = ["timings", "tap", "checkpoint", "memory", ""]
156 #cmd_list = ["tap"]
157 try:
158 for cmd in cmd_list:
159 node_stats = mc.stats(cmd)
160 if node_stats:
161 if cmd == "timings":
162 # need to preprocess histogram data first
163 vals = sorted([self.seg(*kv) for kv in node_stats.items()])
164 dd = {}
165 totals = {}
166 longest = 0
167 for s in vals:
168 avg = (s[0][1] + s[0][2]) / 2
169 k = s[0][0]
170 l = dd.get(k, [])
171 l.append((avg, s[1]))
172 dd[k] = l
173 totals[k] = totals.get(k, 0) + s[1]
174 for k in sorted(dd):
175 ccount = 0
176 for lbl,v in dd[k]:
177 ccount += v * lbl
178 stats[k] = ccount / totals[k]
179 else:
180 for key, val in node_stats.items():
181 stats[key] = val
182 except Exception, err:
183 traceback.print_exc()
184
185 def get_mc_stats(self, server, bucketlist, nodes):
186 #print util.pretty_print(bucketlist)
187 for bucket in bucketlist:
188 bucket_name = bucket['name']
189 stats_buffer.node_stats[bucket_name] = {}
190 for node in nodes:
191 (node_server, node_port) = util.hostport(node['hostname'])
192 self.log.info(" node: %s %s" % (node_server, node['ports']['direct']))
193 stats = {}
194 mc = mc_bin_client.MemcachedClient(node_server, node['ports']['direct'])
195 if bucket["name"] != "Default":
196 mc.sasl_auth_plain(bucket_name.encode("utf8"), bucket["saslPassword"].encode("utf8"))
197 self.get_mc_stats_per_node(mc, stats)
198 stats_buffer.node_stats[bucket_name][node['hostname']] = stats
199
200 def get_ns_stats(self, bucketlist, server, port, user, password, opts):
201 for bucket in bucketlist:
202 bucket_name = bucket['name']
203 stats_buffer.buckets[bucket_name] = copy.deepcopy(stats_buffer.stats)
204 cmd = 'bucket-node-stats'
205 for scale, stat_set in stats_buffer.buckets[bucket_name].iteritems():
206 for stat in stat_set.iterkeys():
207 sys.stderr.write('.')
208 self.log.debug("retrieve: %s" % stat)
209 c = buckets.BucketNodeStats(bucket_name, stat, scale)
210
211 json = c.runCmd('bucket-node-stats', server, port, user, password, opts)
212 stats_buffer.buckets[bucket_name][scale][stat] = json
213 sys.stderr.write('\n')
214
215 def collect_data(self,cluster, user, password, opts):
216 server, port = util.hostport(cluster)
217
218 #get node list info
219 nodes = self.get_hostlist(server, port, user, password, opts)
220 self.log.debug(util.pretty_print(stats_buffer.nodes))
221
222 #get bucket list
223 bucketlist = self.get_bucketlist(server, port, user, password, opts)
224 self.log.debug(util.pretty_print(stats_buffer.bucket_info))
225
226 #get stats from ep-engine
227 self.get_mc_stats(server, bucketlist, nodes)
228 self.log.debug(util.pretty_print(stats_buffer.node_stats))
229
230 #get stats from ns-server
231 self.get_ns_stats(bucketlist, server, port, user, password, opts)
232 self.log.debug(util.pretty_print(stats_buffer.buckets))
233
Something went wrong with that request. Please try again.