Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Added MySQL metric gathering

  • Loading branch information...
commit 34742fd4c0cb47201505aa942ac16c3b1881d6a4 1 parent 2f3e88f
David Winterbottom authored June 25, 2012
21  bin/pygmetric_mysql
... ...
@@ -0,0 +1,21 @@
  1
+#!/usr/bin/env python
  2
+
  3
+from optparse import OptionParser
  4
+
  5
+from pygmetric import mysql
  6
+
  7
+if __name__ == '__main__':
  8
+    parser = OptionParser()
  9
+    parser.add_option('--user', dest='user', default='/',
  10
+                      help="Specify the user to use")
  11
+    parser.add_option('--password', dest='password', 
  12
+                      help="Specify the password to use")
  13
+    parser.add_option('-d', '--dry-run', dest='debug', default=False,
  14
+                      action='store_true',
  15
+                      help="Print out metric commands without submitting")
  16
+    options, args = parser.parse_args()
  17
+    mysql.submit(
  18
+        user=options.user,
  19
+        password=options.password,
  20
+        debug=options.debug,
  21
+    )
113  pygmetric/mysql.py
... ...
@@ -0,0 +1,113 @@
  1
+import pygmetric.shell
  2
+import pygmetric 
  3
+
  4
+
  5
+def submit(user, password, debug):
  6
+    stats = fetch_stats(user, password)
  7
+    for metric in stats.values():
  8
+        pygmetric.shell.call_gmetric(name=metric['name'],
  9
+                                     value=metric['value'],
  10
+                                     type=metric['type'],
  11
+                                     units=metric['units'],
  12
+                                     debug=debug)
  13
+
  14
+COUNT_METRICS = {
  15
+    'Threads_connected': {
  16
+        'name': 'mysql_threads_connected',
  17
+        'type': 'uint32',
  18
+        'units': 'Threads',
  19
+    },
  20
+    'Threads_running': {
  21
+        'name': 'mysql_threads_running',
  22
+        'type': 'uint32',
  23
+        'units': 'Threads',
  24
+    },
  25
+}
  26
+
  27
+RATE_METRICS = {
  28
+    'Connections': {
  29
+        'name': 'mysql_connections_rate',
  30
+        'type': 'float',
  31
+        'units': 'Conns/second',
  32
+    },
  33
+    'Queries': {
  34
+        'name': 'mysql_query_rate',
  35
+        'type': 'float',
  36
+        'units': 'Queries/second',
  37
+    },
  38
+    'Com_select': {
  39
+        'name': 'mysql_query_select_rate',
  40
+        'type': 'float',
  41
+        'units': 'Queries/second',
  42
+    },
  43
+    'Com_update': {
  44
+        'name': 'mysql_query_update_rate',
  45
+        'type': 'float',
  46
+        'units': 'Queries/second',
  47
+    },
  48
+    'Com_insert': {
  49
+        'name': 'mysql_query_insert',
  50
+        'type': 'float',
  51
+        'units': 'Queries/second',
  52
+    },
  53
+    'Com_delete': {
  54
+        'name': 'mysql_query_delete_rate',
  55
+        'type': 'float',
  56
+        'units': 'Queries/second',
  57
+    },
  58
+    'Com_load': {
  59
+        'name': 'mysql_query_load_rate',
  60
+        'type': 'float',
  61
+        'units': 'Queries/second',
  62
+    },
  63
+    'Slow_queries': {
  64
+        'name': 'mysql_slow_queries',
  65
+        'type': 'float',
  66
+        'units': 'Queries/second',
  67
+    },
  68
+    'Created_tmp_tables': {
  69
+        'name': 'mysql_created_tmp_tables',
  70
+        'type': 'float',
  71
+        'units': 'Tables/second',
  72
+    },
  73
+    'Bytes_received': {
  74
+        'name': 'mysql_bytes_received',
  75
+        'type': 'float',
  76
+        'units': 'Bytes/second',
  77
+    },
  78
+    'Bytes_sent': {
  79
+        'name': 'mysql_bytes_sent',
  80
+        'type': 'float',
  81
+        'units': 'Bytes/second',
  82
+    },
  83
+}
  84
+
  85
+
  86
+def fetch_stats(user, password):
  87
+    cmd = 'mysql --user=%s --password=%s -e "SHOW GLOBAL STATUS"' % (user, password)
  88
+    stdout = pygmetric.shell.run(cmd)
  89
+
  90
+    metrics = {}
  91
+    for line in stdout.split("\n"):
  92
+        parts = line.split()
  93
+        if len(parts) != 5:
  94
+            continue
  95
+        mysql_name, value = parts[1], parts[3]
  96
+        if mysql_name in COUNT_METRICS:
  97
+            prototype = COUNT_METRICS[mysql_name]
  98
+            metrics[prototype['name']] = {
  99
+                'name': prototype['name'],
  100
+                'value': int(value),
  101
+                'type': prototype['type'],
  102
+                'units': prototype['units'],
  103
+            }
  104
+        elif mysql_name in RATE_METRICS:
  105
+            prototype = RATE_METRICS[mysql_name]
  106
+            current_value = int(value)
  107
+            metrics[prototype['name']] = {
  108
+                'name': prototype['name'],
  109
+                'value': pygmetric.get_rate(prototype['name'], current_value, 1),
  110
+                'type': prototype['type'],
  111
+                'units': prototype['units'],
  112
+            }
  113
+    return metrics
2  pygmetric/rabbitmq.py
... ...
@@ -1,5 +1,5 @@
1 1
 import pygmetric.shell
2  
-import pygmetric 
  2
+import pygmetric
3 3
 
4 4
 
5 5
 def submit(vhost, queue, debug):
1  setup.py
@@ -12,4 +12,5 @@
12 12
       scripts=['bin/pygmetric_apache',
13 13
                'bin/pygmetric_nginx',
14 14
                'bin/pygmetric_rabbitmq',
  15
+               'bin/pygmetric_mysql',
15 16
               ])
315  tests/mysql_tests.py
... ...
@@ -0,0 +1,315 @@
  1
+import unittest
  2
+import mock
  3
+
  4
+from pygmetric import mysql
  5
+
  6
+
  7
+STDOUT = """+-----------------------------------+---------------+
  8
+| Aborted_clients                   |         24473 |
  9
+| Aborted_connects                  |            82 |
  10
+| Binlog_cache_disk_use             |         75938 |
  11
+| Binlog_cache_use                  |      58687056 |
  12
+| Bytes_received                    | 1028745350408 |
  13
+| Bytes_sent                        | 3176259802173 |
  14
+| Com_admin_commands                |    2152370079 |
  15
+| Com_assign_to_keycache            |             0 |
  16
+| Com_alter_db                      |             0 |
  17
+| Com_alter_db_upgrade              |             0 |
  18
+| Com_alter_event                   |             0 |
  19
+| Com_alter_function                |             0 |
  20
+| Com_alter_procedure               |             0 |
  21
+| Com_alter_server                  |             0 |
  22
+| Com_alter_table                   |             1 |
  23
+| Com_alter_tablespace              |             0 |
  24
+| Com_analyze                       |             0 |
  25
+| Com_backup_table                  |             0 |
  26
+| Com_begin                         |             0 |
  27
+| Com_binlog                        |             0 |
  28
+| Com_call_procedure                |             0 |
  29
+| Com_change_db                     |            17 |
  30
+| Com_change_master                 |             0 |
  31
+| Com_check                         |             0 |
  32
+| Com_checksum                      |             0 |
  33
+| Com_commit                        |     176495692 |
  34
+| Com_create_db                     |             0 |
  35
+| Com_create_event                  |             0 |
  36
+| Com_create_function               |             0 |
  37
+| Com_create_index                  |             4 |
  38
+| Com_create_procedure              |             0 |
  39
+| Com_create_server                 |             0 |
  40
+| Com_create_table                  |             1 |
  41
+| Com_create_trigger                |             0 |
  42
+| Com_create_udf                    |             0 |
  43
+| Com_create_user                   |             0 |
  44
+| Com_create_view                   |             0 |
  45
+| Com_dealloc_sql                   |             0 |
  46
+| Com_delete                        |        872745 |
  47
+| Com_delete_multi                  |             0 |
  48
+| Com_do                            |             0 |
  49
+| Com_drop_db                       |             0 |
  50
+| Com_drop_event                    |             0 |
  51
+| Com_drop_function                 |             0 |
  52
+| Com_drop_index                    |             0 |
  53
+| Com_drop_procedure                |             0 |
  54
+| Com_drop_server                   |             0 |
  55
+| Com_drop_table                    |             0 |
  56
+| Com_drop_trigger                  |             0 |
  57
+| Com_drop_user                     |             0 |
  58
+| Com_drop_view                     |             0 |
  59
+| Com_empty_query                   |             0 |
  60
+| Com_execute_sql                   |             0 |
  61
+| Com_flush                         |            35 |
  62
+| Com_grant                         |             0 |
  63
+| Com_ha_close                      |             0 |
  64
+| Com_ha_open                       |             0 |
  65
+| Com_ha_read                       |             0 |
  66
+| Com_help                          |             0 |
  67
+| Com_insert                        |      11003204 |
  68
+| Com_insert_select                 |             0 |
  69
+| Com_install_plugin                |             0 |
  70
+| Com_kill                          |             2 |
  71
+| Com_load                          |        266282 |
  72
+| Com_load_master_data              |             0 |
  73
+| Com_load_master_table             |             0 |
  74
+| Com_lock_tables                   |             0 |
  75
+| Com_optimize                      |             0 |
  76
+| Com_preload_keys                  |             0 |
  77
+| Com_prepare_sql                   |             0 |
  78
+| Com_purge                         |             0 |
  79
+| Com_purge_before_date             |             0 |
  80
+| Com_release_savepoint             |        606996 |
  81
+| Com_rename_table                  |             0 |
  82
+| Com_rename_user                   |             0 |
  83
+| Com_repair                        |             0 |
  84
+| Com_replace                       |             0 |
  85
+| Com_replace_select                |             0 |
  86
+| Com_reset                         |             0 |
  87
+| Com_restore_table                 |             0 |
  88
+| Com_revoke                        |             0 |
  89
+| Com_revoke_all                    |             0 |
  90
+| Com_rollback                      |         40464 |
  91
+| Com_rollback_to_savepoint         |          2769 |
  92
+| Com_savepoint                     |      15556478 |
  93
+| Com_select                        |     894856486 |
  94
+| Com_set_option                    |      62692570 |
  95
+| Com_show_authors                  |             0 |
  96
+| Com_show_binlog_events            |             0 |
  97
+| Com_show_binlogs                  |         10030 |
  98
+| Com_show_charsets                 |             0 |
  99
+| Com_show_collations               |             0 |
  100
+| Com_show_column_types             |             0 |
  101
+| Com_show_contributors             |             0 |
  102
+| Com_show_create_db                |             0 |
  103
+| Com_show_create_event             |             0 |
  104
+| Com_show_create_func              |             0 |
  105
+| Com_show_create_proc              |             0 |
  106
+| Com_show_create_table             |            58 |
  107
+| Com_show_create_trigger           |             0 |
  108
+| Com_show_databases                |            51 |
  109
+| Com_show_engine_logs              |             0 |
  110
+| Com_show_engine_mutex             |             0 |
  111
+| Com_show_engine_status            |         10946 |
  112
+| Com_show_events                   |             0 |
  113
+| Com_show_errors                   |             0 |
  114
+| Com_show_fields                   |          9198 |
  115
+| Com_show_function_status          |             0 |
  116
+| Com_show_grants                   |             0 |
  117
+| Com_show_keys                     |            14 |
  118
+| Com_show_master_status            |             2 |
  119
+| Com_show_new_master               |             0 |
  120
+| Com_show_open_tables              |             0 |
  121
+| Com_show_plugins                  |             0 |
  122
+| Com_show_privileges               |             0 |
  123
+| Com_show_procedure_status         |             0 |
  124
+| Com_show_processlist              |           241 |
  125
+| Com_show_profile                  |             0 |
  126
+| Com_show_profiles                 |             0 |
  127
+| Com_show_slave_hosts              |             0 |
  128
+| Com_show_slave_status             |         10023 |
  129
+| Com_show_status                   |         58302 |
  130
+| Com_show_storage_engines          |             0 |
  131
+| Com_show_table_status             |            56 |
  132
+| Com_show_tables                   |            58 |
  133
+| Com_show_triggers                 |             0 |
  134
+| Com_show_variables                |         11002 |
  135
+| Com_show_warnings                 |            55 |
  136
+| Com_slave_start                   |             0 |
  137
+| Com_slave_stop                    |             0 |
  138
+| Com_stmt_close                    |             0 |
  139
+| Com_stmt_execute                  |             0 |
  140
+| Com_stmt_fetch                    |             0 |
  141
+| Com_stmt_prepare                  |             0 |
  142
+| Com_stmt_reprepare                |             0 |
  143
+| Com_stmt_reset                    |             0 |
  144
+| Com_stmt_send_long_data           |             0 |
  145
+| Com_truncate                      |             0 |
  146
+| Com_uninstall_plugin              |             0 |
  147
+| Com_unlock_tables                 |             0 |
  148
+| Com_update                        |     152365778 |
  149
+| Com_update_multi                  |             0 |
  150
+| Com_xa_commit                     |             0 |
  151
+| Com_xa_end                        |             0 |
  152
+| Com_xa_prepare                    |             0 |
  153
+| Com_xa_recover                    |             0 |
  154
+| Com_xa_rollback                   |             0 |
  155
+| Com_xa_start                      |             0 |
  156
+| Compression                       |           OFF |
  157
+| Connections                       |      17691201 |
  158
+| Created_tmp_disk_tables           |       6717413 |
  159
+| Created_tmp_files                 |         65863 |
  160
+| Created_tmp_tables                |      10122367 |
  161
+| Delayed_errors                    |             0 |
  162
+| Delayed_insert_threads            |             0 |
  163
+| Delayed_writes                    |             0 |
  164
+| Flush_commands                    |             1 |
  165
+| Handler_commit                    |    1355062554 |
  166
+| Handler_delete                    |      92281479 |
  167
+| Handler_discover                  |             0 |
  168
+| Handler_prepare                   |     238315960 |
  169
+| Handler_read_first                |      13737350 |
  170
+| Handler_read_key                  |    5510384504 |
  171
+| Handler_read_next                 |  777739791735 |
  172
+| Handler_read_prev                 |     269264307 |
  173
+| Handler_read_rnd                  |     259099631 |
  174
+| Handler_read_rnd_next             | 1574207601568 |
  175
+| Handler_rollback                  |        388151 |
  176
+| Handler_savepoint                 |       9695864 |
  177
+| Handler_savepoint_rollback        |          2769 |
  178
+| Handler_update                    |     165655218 |
  179
+| Handler_write                     |     172316650 |
  180
+| Innodb_buffer_pool_pages_data     |        377286 |
  181
+| Innodb_buffer_pool_pages_dirty    |          2573 |
  182
+| Innodb_buffer_pool_pages_flushed  |      90612660 |
  183
+| Innodb_buffer_pool_pages_free     |             2 |
  184
+| Innodb_buffer_pool_pages_misc     |         15928 |
  185
+| Innodb_buffer_pool_pages_total    |        393216 |
  186
+| Innodb_buffer_pool_read_ahead_rnd |       1567483 |
  187
+| Innodb_buffer_pool_read_ahead_seq |        825008 |
  188
+| Innodb_buffer_pool_read_requests  |  739476600066 |
  189
+| Innodb_buffer_pool_reads          |     103962801 |
  190
+| Innodb_buffer_pool_wait_free      |             0 |
  191
+| Innodb_buffer_pool_write_requests |    3353944439 |
  192
+| Innodb_data_fsyncs                |       6497162 |
  193
+| Innodb_data_pending_fsyncs        |             1 |
  194
+| Innodb_data_pending_reads         |             0 |
  195
+| Innodb_data_pending_writes        |             0 |
  196
+| Innodb_data_read                  | 3079891750912 |
  197
+| Innodb_data_reads                 |     123682976 |
  198
+| Innodb_data_writes                |     297951623 |
  199
+| Innodb_data_written               | 3249276207616 |
  200
+| Innodb_dblwr_pages_written        |      90612788 |
  201
+| Innodb_dblwr_writes               |       1338028 |
  202
+| Innodb_log_waits                  |          3367 |
  203
+| Innodb_log_write_requests         |     366261719 |
  204
+| Innodb_log_writes                 |     224235779 |
  205
+| Innodb_os_log_fsyncs              |       3822255 |
  206
+| Innodb_os_log_pending_fsyncs      |             0 |
  207
+| Innodb_os_log_pending_writes      |             0 |
  208
+| Innodb_os_log_written             |  279872190976 |
  209
+| Innodb_page_size                  |         16384 |
  210
+| Innodb_pages_created              |       3612634 |
  211
+| Innodb_pages_read                 |     187981046 |
  212
+| Innodb_pages_written              |      90612788 |
  213
+| Innodb_row_lock_current_waits     |             0 |
  214
+| Innodb_row_lock_time              |     440567084 |
  215
+| Innodb_row_lock_time_avg          |          2077 |
  216
+| Innodb_row_lock_time_max          |        330631 |
  217
+| Innodb_row_lock_waits             |        212095 |
  218
+| Innodb_rows_deleted               |      92281457 |
  219
+| Innodb_rows_inserted              |     118011078 |
  220
+| Innodb_rows_read                  | 2349244437117 |
  221
+| Innodb_rows_updated               |     163641219 |
  222
+| Key_blocks_not_flushed            |             0 |
  223
+| Key_blocks_unused                 |         13396 |
  224
+| Key_blocks_used                   |          2220 |
  225
+| Key_read_requests                 |      33263586 |
  226
+| Key_reads                         |             3 |
  227
+| Key_write_requests                |       5293445 |
  228
+| Key_writes                        |             0 |
  229
+| Last_query_cost                   |      0.000000 |
  230
+| Max_used_connections              |           301 |
  231
+| Not_flushed_delayed_rows          |             0 |
  232
+| Open_files                        |             4 |
  233
+| Open_streams                      |             0 |
  234
+| Open_table_definitions            |           210 |
  235
+| Open_tables                       |            64 |
  236
+| Opened_files                      |      28736956 |
  237
+| Opened_table_definitions          |           225 |
  238
+| Opened_tables                     |      11535472 |
  239
+| Prepared_stmt_count               |             0 |
  240
+| Qcache_free_blocks                |           863 |
  241
+| Qcache_free_memory                |       6111120 |
  242
+| Qcache_hits                       |    1079435027 |
  243
+| Qcache_inserts                    |     500404683 |
  244
+| Qcache_lowmem_prunes              |     320043363 |
  245
+| Qcache_not_cached                 |     394613750 |
  246
+| Qcache_queries_in_cache           |          6987 |
  247
+| Qcache_total_blocks               |         14878 |
  248
+| Queries                           |    2411941540 |
  249
+| Questions                         |    2411941540 |
  250
+| Rpl_status                        |          NULL |
  251
+| Select_full_join                  |            56 |
  252
+| Select_full_range_join            |             0 |
  253
+| Select_range                      |      23096318 |
  254
+| Select_range_check                |             0 |
  255
+| Select_scan                       |      13775608 |
  256
+| Slave_open_temp_tables            |             0 |
  257
+| Slave_retried_transactions        |             0 |
  258
+| Slave_running                     |           OFF |
  259
+| Slow_launch_threads               |             0 |
  260
+| Slow_queries                      |        204648 |
  261
+| Sort_merge_passes                 |        395205 |
  262
+| Sort_range                        |      21797651 |
  263
+| Sort_rows                         |    1007504249 |
  264
+| Sort_scan                         |      12725760 |
  265
+| Ssl_accept_renegotiates           |             0 |
  266
+| Ssl_accepts                       |             0 |
  267
+| Ssl_callback_cache_hits           |             0 |
  268
+| Ssl_cipher                        |               |
  269
+| Ssl_cipher_list                   |               |
  270
+| Ssl_client_connects               |             0 |
  271
+| Ssl_connect_renegotiates          |             0 |
  272
+| Ssl_ctx_verify_depth              |             0 |
  273
+| Ssl_ctx_verify_mode               |             0 |
  274
+| Ssl_default_timeout               |             0 |
  275
+| Ssl_finished_accepts              |             0 |
  276
+| Ssl_finished_connects             |             0 |
  277
+| Ssl_session_cache_hits            |             0 |
  278
+| Ssl_session_cache_misses          |             0 |
  279
+| Ssl_session_cache_mode            |          NONE |
  280
+| Ssl_session_cache_overflows       |             0 |
  281
+| Ssl_session_cache_size            |             0 |
  282
+| Ssl_session_cache_timeouts        |             0 |
  283
+| Ssl_sessions_reused               |             0 |
  284
+| Ssl_used_session_cache_entries    |             0 |
  285
+| Ssl_verify_depth                  |             0 |
  286
+| Ssl_verify_mode                   |             0 |
  287
+| Ssl_version                       |               |
  288
+| Table_locks_immediate             |    1284767674 |
  289
+| Table_locks_waited                |             3 |
  290
+| Tc_log_max_pages_used             |             0 |
  291
+| Tc_log_page_size                  |             0 |
  292
+| Tc_log_page_waits                 |             9 |
  293
+| Threads_cached                    |             3 |
  294
+| Threads_connected                 |            15 |
  295
+| Threads_created                   |        371342 |
  296
+| Threads_running                   |             3 |
  297
+| Uptime                            |       3036021 |
  298
+| Uptime_since_flush_status         |       3036021 |
  299
++-----------------------------------+---------------+"""
  300
+
  301
+
  302
+@mock.patch('pygmetric.shell.run')
  303
+class MySQLTests(unittest.TestCase):
  304
+
  305
+    def test_threads_connected(self, mock_run):
  306
+        mock_run.return_value = STDOUT
  307
+        stats = mysql.fetch_stats(user='ganglia', password='cheese')
  308
+        self.assertEqual(stats['mysql_threads_connected']['value'], 15)
  309
+
  310
+    @mock.patch('pygmetric.get_rate')
  311
+    def test_connections_rate_captured(self, mock_rate, mock_run):
  312
+        mock_run.return_value = STDOUT
  313
+        mock_rate.return_value = 3
  314
+        stats = mysql.fetch_stats(user='ganglia', password='cheese')
  315
+        self.assertEqual(stats['mysql_connections_rate']['value'], 3)

0 notes on commit 34742fd

Please sign in to comment.
Something went wrong with that request. Please try again.