Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Initial public version

  • Loading branch information...
commit f6f22031c5af7a84b1855646c31a10ec257e7be9 0 parents
Dennis Kaarsemaker authored September 27, 2012
8  MySQLPerfCollector.conf
... ...
@@ -0,0 +1,8 @@
  1
+enabled = True
  2
+interval = 5
  3
+host = my-name.my-domain
  4
+db = performance_schema
  5
+user = graphite
  6
+passwd = SomePass
  7
+slave = True
  8
+innodb = True
158  MySQLPerfCollector.py
... ...
@@ -0,0 +1,158 @@
  1
+# Diamond collector that monitors relevant MySQL performance_schema values
  2
+# For now only monitors replication load
  3
+#
  4
+# (c) 2012 Dennis Kaarsemaker <dennis@kaarsemaker.net>
  5
+
  6
+from __future__ import division
  7
+
  8
+try:
  9
+    import MySQLdb
  10
+    from MySQLdb import MySQLError
  11
+except ImportError:
  12
+    MySQLdb = None
  13
+import diamond
  14
+import re
  15
+import time
  16
+
  17
+class MySQLPerfCollector(diamond.collector.Collector):
  18
+
  19
+    def __init__(self, *args, **kwargs):
  20
+        super(MySQLPerfCollector,self).__init__(*args, **kwargs)
  21
+        self.db = None
  22
+        self.cursor = None
  23
+        self.connect()
  24
+        self.last_wait_count = {}
  25
+        self.last_wait_sum = {}
  26
+        self.last_timestamp = {}
  27
+        self.last_data = {}
  28
+        self.monitors = {
  29
+            'slave_sql': {
  30
+                'wait/synch/cond/sql/MYSQL_RELAY_LOG::update_cond': 'wait_for_update',
  31
+                'wait/io/file/innodb/innodb_data_file':             'innodb_data_file',
  32
+                'wait/io/file/innodb/innodb_log_file':              'innodb_log_file',
  33
+                'wait/io/file/myisam/dfile':                        'myisam_dfile',
  34
+                'wait/io/file/myisam/kfile':                        'myisam_kfile',
  35
+                'wait/io/file/sql/binlog':                          'binlog',
  36
+                'wait/io/file/sql/relay_log_info':                  'relaylog_info',
  37
+                'wait/io/file/sql/relaylog':                        'relaylog',
  38
+                'wait/synch/mutex/innodb':                          'innodb_mutex',
  39
+                'wait/synch/mutex':                                 'other_mutex',
  40
+                'wait/synch/rwlock':                                'rwlocks',
  41
+                'wait/io':                                          'other_io',
  42
+            },
  43
+            'slave_io': {
  44
+                'wait/io/file/sql/relaylog_index':                  'relaylog_index',
  45
+                'wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_index': 'relaylog_index_lock',
  46
+                'wait/synch/mutex/sql/Master_info::data_lock':      'master_info_lock',
  47
+                'wait/synch/mutex/mysys/IO_CACHE::append_buffer_lock': 'append_buffer_lock',
  48
+                'wait/synch/mutex/sql/LOG::LOCK_log':               'log_lock',
  49
+                'wait/io/file/sql/master_info':                     'master_info',
  50
+                'wait/io/file/sql/relaylog':                        'relaylog',
  51
+                'wait/synch/mutex':                                 'other_mutex',
  52
+                'wait/synch/rwlock':                                'rwlocks',
  53
+                'wait/io':                                          'other_io',
  54
+            }
  55
+        }
  56
+
  57
+    def get_default_config(self):
  58
+        """
  59
+        Returns the default collector settings
  60
+        """
  61
+        return {
  62
+            'path':     'mysql',
  63
+            # Connection settings
  64
+            'host':     'localhost',
  65
+            'port':     3306,
  66
+            'db':       'yourdatabase',
  67
+            'user':     'yourusername',
  68
+            'passwd':   'yourpassword',
  69
+            
  70
+            'slave':    'False',
  71
+            'master':   'False',
  72
+            'innodb':    'False',
  73
+        }
  74
+
  75
+    def connect(self):
  76
+        if MySQLdb is None:
  77
+            self.log.error('Unable to import MySQLdb')
  78
+            return
  79
+
  80
+        params = {}
  81
+        params['host']   = self.config['host']
  82
+        params['port']   = int(self.config['port'])
  83
+        params['db']     = self.config['db']
  84
+        params['user']   = self.config['user']
  85
+        params['passwd'] = self.config['passwd']
  86
+
  87
+        try:
  88
+            self.db = MySQLdb.connect(**params)
  89
+        except MySQLError, e:
  90
+            self.log.error('MySQLPerfCollector couldnt connect to database %s', e)
  91
+            return {}
  92
+
  93
+        self.log.info('MySQLPerfCollector: Connected to database.')
  94
+
  95
+    def slave_load(self, thread):
  96
+        cursor = self.db.cursor()
  97
+        cursor.execute("""
  98
+            SELECT 
  99
+                his.event_name,
  100
+                his.sum_timer_wait,
  101
+                his.count_star, 
  102
+                cur.event_name, 
  103
+                UNIX_TIMESTAMP(SYSDATE())
  104
+            FROM 
  105
+                events_waits_summary_by_thread_by_event_name his
  106
+                JOIN threads thr USING (thread_id)
  107
+                JOIN events_waits_current cur USING (thread_id)
  108
+            WHERE
  109
+                name = %s
  110
+            ORDER BY
  111
+                his.event_name
  112
+            """, (thread,))
  113
+
  114
+        data = list(cursor.fetchall())
  115
+        wait_sum = sum([x[1] for x in data])
  116
+        wait_count = sum([x[2] for x in data])
  117
+        cur_event_name, timestamp = data[0][3:] 
  118
+        load = {}
  119
+
  120
+        if thread not in self.last_wait_sum:
  121
+            # Avoid bogus data
  122
+            self.last_wait_sum[thread], self.last_wait_count[thread], self.last_timestamp[thread] = \
  123
+                wait_sum, wait_count, timestamp
  124
+            self.last_data[thread] = data
  125
+            return
  126
+
  127
+        wait_delta = wait_sum - self.last_wait_sum[thread]
  128
+        time_delta = (timestamp - self.last_timestamp[thread]) * 1000000000000;
  129
+
  130
+        # Summarize a few things
  131
+        thread_name = thread[thread.rfind('/')+1:]
  132
+        data.append(['wait/synch/mutex/innodb', sum([x[1] for x in data if x[0].startswith('wait/synch/mutex/innodb')])])
  133
+        data.append(['wait/synch/mutex', sum([x[1] for x in data if x[0].startswith('wait/synch/mutex') and x[0] not in self.monitors[thread_name]]) - data[-1][1]])
  134
+        data.append(['wait/synch/rwlock', sum([x[1] for x in data if x[0].startswith('wait/synch/rwlock')])])
  135
+        data.append(['wait/io', sum([x[1] for x in data if x[0].startswith('wait/io') and x[0] not in self.monitors[thread_name]])])
  136
+
  137
+
  138
+        for d in zip(self.last_data[thread], data):
  139
+            if d[0][0] in self.monitors[thread_name]:
  140
+                self.publish(thread_name + '.' + self.monitors[thread_name][d[0][0]], (d[1][1] - d[0][1])/time_delta * 100)
  141
+
  142
+        # Also log what's unaccounted for. This is where Actual Work gets done
  143
+        self.publish(thread_name + '.other_work',  float(time_delta - wait_delta) / time_delta * 100)
  144
+
  145
+        self.last_wait_sum[thread], self.last_wait_count[thread], self.last_timestamp[thread] = \
  146
+            wait_sum, wait_count, timestamp
  147
+        self.last_data[thread] = data
  148
+
  149
+
  150
+    def collect(self):
  151
+        if self.config['slave']:
  152
+            try:
  153
+                self.slave_load('thread/sql/slave_io')
  154
+                self.slave_load('thread/sql/slave_sql')
  155
+            except MySQLdb.OperationalError:
  156
+                self.connect()
  157
+                self.slave_load('thread/sql/slave_io')
  158
+                self.slave_load('thread/sql/slave_sql')
11  README
... ...
@@ -0,0 +1,11 @@
  1
+This is a MySQL performance_schema monitor to be used by diamond
  2
+(http://opensource.brightcove.com/project/diamond/) to collect data for
  3
+graphite (http://graphite.readthedocs.org).
  4
+
  5
+At the moment it only collects replication load information, giving you insight
  6
+in how loaded your single-threaded mysql replication is.
  7
+
  8
+etsy-snippet.php is a snippet for an etsy-based dashboard that combines the
  9
+metrics to a single graph, an example of which is included as well.
  10
+
  11
+(c) 2012, Dennis Kaarsemaker <dennis@kaarsemaker.net>
34  etsy-snippet.php
... ...
@@ -0,0 +1,34 @@
  1
+<?php
  2
+
  3
+$sql_metrics = array(
  4
+    array("binlog", "Binlog IO"),
  5
+    array("innodb_data_file", "InnoDB data IO"),
  6
+    array("innodb_log_file", "InnoDB log IO"),
  7
+    array("innodb_mutex", "InnoDB locks"),
  8
+    array("myisam_dfile", "MyISAM data IO"),
  9
+    array("myisam_kfile", "MyISAM index IO"),
  10
+    array("relaylog", "Relaylog IO"),
  11
+    array("relaylog_info", "Relaylog info IO"),
  12
+    array("other_io", "Other IO"),
  13
+    array("other_mutex", "Other mutexes"),
  14
+    array("rwlocks", "Other rwlocks"),
  15
+    array("other_work", "Processing query"),
  16
+    array("wait_for_update", "Idle"),
  17
+);
  18
+
  19
+$host = "my-database-server_example_com";
  20
+foreach($hosts as $host) {
  21
+    $g = new Graphite($time);
  22
+
  23
+    $g->setTitle("Replication load on $host (SQL thread)");
  24
+    $g->setVTitle("");
  25
+    foreach($sql_metrics as $metric) {
  26
+      $g->addMetric("alias(sys.$host.mysql.slave_sql.${metric[0]}, \"${metric[1]}\")");
  27
+    }
  28
+    $g->displayStacked(True);
  29
+    $g->setYMin(0);
  30
+    $g->setYMax(105);
  31
+    print($g->getDashboardHTMLmod(1000,500,$hostsearch);
  32
+
  33
+}
  34
+?>
BIN  replication_load.png

0 notes on commit f6f2203

Please sign in to comment.
Something went wrong with that request. Please try again.