Permalink
Browse files

Merge pull request #51 from diclophis/master

Add MySQL Thread Pool Plugin Monitoring
  • Loading branch information...
2 parents cb82065 + 285de1e commit fba3b6085b9db04d1c76bda1c9a85514f3106839 @itsderek23 itsderek23 committed Mar 26, 2012
@@ -0,0 +1,31 @@
+TP_GROUP_ID CONNECTIONS_STARTED CONNECTIONS_CLOSED QUERIES_EXECUTED QUERIES_QUEUED THREADS_STARTED PRIO_KICKUPS STALLED_QUERIES_EXECUTED BECOME_CONSUMER_THREAD BECOME_RESERVE_THREAD BECOME_WAITING_THREAD WAKE_THREAD_STALL_CHECKER SLEEP_WAITS DISK_IO_WAITS ROW_LOCK_WAITS GLOBAL_LOCK_WAITS META_DATA_LOCK_WAITS TABLE_LOCK_WAITS USER_LOCK_WAITS BINLOG_WAITS GROUP_COMMIT_WAITS FSYNC_WAITS
+0 4265 4246 206888317 118563783 23 0 16696 5726735 904472 138759896 21719 0 6616163 214 0 0 234 0 0 0 0
+1 4265 4243 197027460 107754480 20 0 14973 4966602 780203 127384531 19476 0 5735291 242 0 0 214 0 0 0 0
+2 4265 4241 202125891 113832010 20 0 16439 5418128 888513 133583879 21271 0 6300225 216 0 1 240 0 0 0 0
+3 4265 4242 203596199 116195124 22 0 16795 5682096 865833 135991238 21767 0 6547860 306 0 1 267 0 0 0 0
+4 4264 4245 206353483 114575793 23 0 16480 5610004 904148 135339083 21091 0 6499259 499 0 0 262 0 0 0 0
+5 4264 4242 200705950 112183615 22 0 16174 5377575 913677 131723280 21000 0 6276228 215 0 0 225 0 0 0 0
+6 4264 4244 197110764 105863773 22 0 15613 5554239 870237 126742657 20291 0 6411987 224 0 1 253 0 0 0 0
+7 4264 4237 202323563 111040644 20 0 16576 5509700 943808 132052800 21671 0 6438238 208 0 0 213 0 0 0 0
+8 4264 4243 205112517 117522195 23 0 16924 5532511 866953 137514250 21701 0 6390029 387 0 0 255 0 0 0 0
+9 4264 4233 201429427 107197308 23 0 16288 5547958 911855 128746366 21045 0 6446276 551 0 0 247 0 0 0 0
+10 4264 4242 209035869 119341649 25 0 16989 5723051 889879 139341900 21997 0 6632674 713 0 0 243 0 0 0 0
+11 4264 4239 205167828 114105172 21 0 16549 6446455 835610 134139425 21553 0 7313211 465 0 1 178 0 0 0 0
+12 4264 4242 203955856 113011841 21 0 16630 7405630 938628 133977566 21273 0 8331258 880 0 1 292 0 0 0 0
+13 4264 4236 205150483 116121166 21 0 16871 5302912 862559 136101239 21787 0 6151233 557 0 0 250 0 0 0 0
+14 4264 4236 213307288 121772081 21 0 17437 10897712 980422 141943665 22650 0 11891658 321 0 0 254 0 0 0 0
+15 4264 4237 199408024 109983030 22 0 16017 6407699 783625 129823124 20684 0 7418288 381 0 1 226 0 0 0 0
+16 4264 4243 200401293 113101368 26 0 16309 5303099 810151 133009334 21132 0 6102571 635 0 1 204 0 0 0 0
+17 4264 4239 212201358 120803104 26 0 17023 5614626 906600 141550519 21794 0 6505402 500 0 3 239 0 0 0 0
+18 4264 4235 203375887 114761170 22 0 16321 7248203 912173 135132996 21156 0 8342319 311 0 0 220 0 0 0 0
+19 4264 4238 202731500 116779212 23 0 16448 5075118 784412 135802524 21257 0 5845923 234 0 2 217 0 0 0 0
+20 4264 4240 204308294 115538791 23 0 16311 6001869 918242 135814637 21077 0 6909163 265 0 1 263 0 0 0 0
+21 4264 4238 216219952 125213752 22 0 17536 6511513 978380 145932963 22582 0 7485472 291 0 1 267 0 0 0 0
+22 4264 4245 207225395 120187323 24 0 16493 5499674 893060 140173470 21312 0 6377232 284 0 1 281 0 0 0 0
+23 4264 4238 207003431 118674005 25 0 16331 5909540 826496 138205513 21230 0 6723020 306 0 3 208 0 0 0 0
+24 4264 4242 199188567 109869932 26 0 15626 7255943 857525 130498161 20395 0 8104707 217 0 2 203 0 0 0 0
+25 4264 4240 212240792 126629263 27 0 16716 5699223 905715 145864340 21680 0 6598282 296 0 1 254 0 0 0 0
+26 4264 4241 208561338 116815417 22 0 17173 5969748 941983 137822945 22327 0 6906510 273 0 3 268 0 0 0 0
+27 4264 4240 202294560 116465968 22 0 17146 5247234 838054 135649835 22043 0 6070917 268 0 0 233 0 0 0 0
+28 4264 4248 194010014 107215223 20 0 15417 5342397 864245 127643520 20101 0 6193324 169 0 1 217 0 0 0 0
+29 4264 4243 201400189 114444785 22 0 16189 5268176 830789 133380416 20837 0 6083951 256 0 0 249 0 0 0 0
@@ -0,0 +1,153 @@
+# Monitors MySQL thread pool plugin
+#
+# Charts the rate-of-change of the important fields from the INFORMATION_SCHEMA TP_THREAD_GROUP_STATS Table
+# Sudden spikes indicate abnormal events esp. regarding the *_WAITS fields
+#
+# http://dev.mysql.com/doc/refman/5.5/en/thread-pool-plugin.html
+# http://dev.mysql.com/doc/refman/5.5/en/thread-pool-tuning.html
+
+# The INFORMATION_SCHEMA TP_THREAD_GROUP_STATS Table
+
+# This table reports statistics per thread group. There is one row per group. The table has these columns:
+
+# TP_GROUP_ID
+# The thread group ID. This is a unique key within the table.
+
+# CONNECTIONS_STARTED
+# The number of connections started.
+
+# CONNECTIONS_CLOSED
+# The number of connections closed.
+
+# QUERIES_EXECUTED
+# The number of statements executed. This number is incremented when a statement starts executing, not when it finishes.
+
+# QUERIES_QUEUED
+# The number of statements received that were queued for execution. This does not count statements that the thread group was able to begin executing immediately without queuing, which can happen under the conditions described in Section 7.11.6.2, “Thread Pool Operation”.
+
+# THREADS_STARTED
+# The number of threads started.
+
+# PRIO_KICKUPS
+# The number of statements that have been moved from low-priority queue to high-priority queue based on the value of the thread_pool_prio_kickup_timer system variable. If this number increases quickly, consider increasing the value of that variable. A quickly increasing counter means that the priority system is not keeping transactions from starting too early. For InnoDB, this most likely means deteriorating performance due to too many concurrent transactions..
+
+# STALLED_QUERIES_EXECUTED
+# The number of statements that have become defined as stalled due to executing for a time longer than the value of the thread_pool_stall_limit system variable.
+
+# BECOME_CONSUMER_THREAD
+# The number of times thread have been assigned the consumer thread role.
+
+# BECOME_RESERVE_THREAD
+# The number of times threads have been assigned the reserve thread role.
+
+# BECOME_WAITING_THREAD
+# The number of times threads have been assigned the waiter thread role. When statements are queued, this happens very often, even in normal operation, so rapid increases in this value are normal in the case of a highly loaded system where statements are queued up.
+
+# WAKE_THREAD_STALL_CHECKER
+# The number of times the stall check thread decided to wake or create a thread to possibly handle some statements or take care of the waiter thread role.
+
+# SLEEP_WAITS
+# The number of THD_WAIT_SLEEP waits. These occur when threads go to sleep; for example, by calling the SLEEP() function.
+
+# DISK_IO_WAITS
+# The number of THD_WAIT_DISKIO waits. These occur when threads perform disk I/O that is likely to not hit the file system cache. Such waits occur when the buffer pool reads and writes data to disk, not for normal reads from and writes to files.
+
+# ROW_LOCK_WAITS
+# The number of THD_WAIT_ROW_LOCK waits for release of a row lock by another transaction.
+
+# GLOBAL_LOCK_WAITS
+# The number of THD_WAIT_GLOBAL_LOCK waits for a global lock to be released.
+
+# META_DATA_LOCK_WAITS
+# The number of THD_WAIT_META_DATA_LOCK waits for a metadata lock to be released.
+
+# TABLE_LOCK_WAITS
+# The number of THD_WAIT_TABLE_LOCK waits for a table to be unlocked that the statement needs to access.
+
+# USER_LOCK_WAITS
+# The number of THD_WAIT_USER_LOCK waits for a special lock constructed by the user thread.
+
+# BINLOG_WAITS
+# The number of THD_WAIT_BINLOG_WAITS waits for the binary log to become free.
+
+# GROUP_COMMIT_WAITS
+# The number of THD_WAIT_GROUP_COMMIT waits. These occur when a group commit must wait for the other parties to complete their part of a transaction.
+
+# FSYNC_WAITS
+# The number of THD_WAIT_SYNC waits for a file sync operation.
+
+
+require 'time'
+require 'date'
+
+class MysqlThreadPoolMonitor < Scout::Plugin
+
+ needs 'mysql'
+
+ OPTIONS=<<-EOS
+ host:
+ name: Host
+ notes: The host to monitor
+ default: 127.0.0.1
+ port:
+ name: Port
+ notes: The port number on the host
+ default: 3306
+ username:
+ name: Username
+ notes: The MySQL username to use
+ default: root
+ password:
+ name: Password
+ notes: The password for the mysql user
+ default:
+ attributes: password
+ EOS
+
+ HEADERS_TO_TRACK_RATE_OF = %w{CONNECTIONS_STARTED CONNECTIONS_CLOSED QUERIES_QUEUED THREADS_STARTED PRIO_KICKUPS STALLED_QUERIES_EXECUTED BECOME_CONSUMER_THREAD BECOME_RESERVE_THREAD BECOME_WAITING_THREAD WAKE_THREAD_STALL_CHECKER SLEEP_WAITS DISK_IO_WAITS ROW_LOCK_WAITS GLOBAL_LOCK_WAITS META_DATA_LOCK_WAITS TABLE_LOCK_WAITS USER_LOCK_WAITS BINLOG_WAITS GROUP_COMMIT_WAITS FSYNC_WAITS}
+
+ SELECT_THREAD_GROUP_STATS = "SELECT * FROM information_schema.TP_THREAD_GROUP_STATS"
+
+ def build_report
+ begin
+ connection = Mysql.new(option(:host) || "localhost", option(:username) || "root", option(:password), nil, option(:port).to_i)
+ result = connection.query(SELECT_THREAD_GROUP_STATS)
+ thread_group_stats = fetch_all_rows_as_hash(result)
+ if thread_group_stats.length > 0 then
+ sum_counts = {}
+ thread_group_count = thread_group_stats.length
+ thread_group_stats.each { |row|
+ HEADERS_TO_TRACK_RATE_OF.each { |tracked_header|
+ tracked_value = row[tracked_header].to_i
+ if sum_counts[tracked_header] then
+ sum_counts[tracked_header] += tracked_value
+ else
+ sum_counts[tracked_header] = tracked_value
+ end
+ }
+ }
+
+ sum_counts.each { |key, value|
+ counter("#{key.downcase}_rate_avg", (value.to_f / thread_group_count.to_f).to_i, :per => :minute, :round => true)
+ }
+ end
+ rescue Mysql::Error => e
+ if e.to_s.include?("Unknown table 'tp_thread_group_stats'") then
+ error("MySQL thread pool plugin not installed", e.to_s)
+ else
+ error("Unable to connect to MySQL", e.to_s)
+ end
+ end
+ end
+
+private
+
+ def fetch_all_rows_as_hash(result)
+ rows = []
+ while row = result.fetch_hash
+ rows << row
+ end
+ rows
+ end
+
+end
@@ -0,0 +1,121 @@
+require File.expand_path('../../test_helper.rb', __FILE__)
+require File.expand_path('../mysql_thread_pool_monitor.rb', __FILE__)
+
+require 'mysql'
+
+class MysqlThreadPoolMonitorTest < Test::Unit::TestCase
+
+ def setup
+ @options = parse_defaults("mysql_thread_pool_monitor")
+ end
+
+ # Tests that if the plugin is able to connect, and process the SELECT_THREAD_GROUP_STATS no errors are returned
+ def test_connect
+ ms_res = Mysql::Result.new
+ ms_res.stubs(:fetch_hash).returns({}, nil)
+ mock_mysql_connection.stubs(:query).with(MysqlThreadPoolMonitor::SELECT_THREAD_GROUP_STATS).returns(ms_res).once
+ plugin = MysqlThreadPoolMonitor.new(nil, memory = {}, @options)
+ result = plugin.run
+ assert result.is_a?(Hash)
+ assert_equal 0, result[:errors].length, "number of errors"
+ end
+
+ # Tests the connection failure error in the event that mysql is not running, or the plugin is mis-configured
+ def test_connect_fail
+ fake_mysql_connect_exception = Mysql::Error.new("Can't connect to MySQL server on '127.0.0.1' (61)")
+ Mysql.stubs(:new).raises(fake_mysql_connect_exception)
+ plugin = MysqlThreadPoolMonitor.new(nil, memory = {}, @options)
+ result = plugin.run
+ assert result.is_a?(Hash)
+ assert_equal 1, result[:errors].length, "number of errors"
+ assert_equal "Unable to connect to MySQL", result[:errors][0][:subject], "Error subject"
+ end
+
+ # Tests the missing tp_thread_group_stats table in the event that the thread_pool plugin is not loaded
+ def test_thread_pool_plugin_not_installed
+ fake_mysql_unknown_table_exception = Mysql::Error.new("Unknown table 'tp_thread_group_stats' in information_schema")
+ mock_mysql = mock_mysql_connection
+ mock_mysql.stubs(:query).with(MysqlThreadPoolMonitor::SELECT_THREAD_GROUP_STATS).raises(fake_mysql_unknown_table_exception)
+ plugin = MysqlThreadPoolMonitor.new(nil, memory = {}, @options)
+ result = plugin.run
+ assert result.is_a?(Hash)
+ assert_equal 1, result[:errors].length, "number of errors"
+ assert_equal "MySQL thread pool plugin not installed", result[:errors][0][:subject], "Error subject"
+ end
+
+ # Tests the expected return value from the fixture is in the expected Array of Hashes format
+ def test_thread_group_stats_fixture
+ fake_result_set = create_thread_group_stats_fixture(0, 0)
+ headers = %w{TP_GROUP_ID CONNECTIONS_STARTED CONNECTIONS_CLOSED QUERIES_EXECUTED QUERIES_QUEUED THREADS_STARTED PRIO_KICKUPS STALLED_QUERIES_EXECUTED BECOME_CONSUMER_THREAD BECOME_RESERVE_THREAD BECOME_WAITING_THREAD WAKE_THREAD_STALL_CHECKER SLEEP_WAITS DISK_IO_WAITS ROW_LOCK_WAITS GLOBAL_LOCK_WAITS META_DATA_LOCK_WAITS TABLE_LOCK_WAITS USER_LOCK_WAITS BINLOG_WAITS GROUP_COMMIT_WAITS FSYNC_WAITS}
+ values = %w{0 4265 4246 206888317 118563783 23 0 16696 5726735 904472 138759896 21719 0 6616163 214 0 0 234 0 0 0 0}
+ first_fake_result = fake_result_set.shift
+ headers.each_with_index { |header, i|
+ assert_equal values[i].to_i, first_fake_result[header]
+ }
+ end
+
+ # Tests the functionality of the plugin, that it reports rate-of-change using the plugin :counter api, and that the desired fields are included in the plugins output
+ def test_plugin_averages_rate_of_change_for_all_important_counters
+ ms_res = Mysql::Result.new
+ min_rate = 0.0
+ max_rate = 10.0
+ ms_res.stubs(:fetch_hash).returns(*create_thread_group_stats_fixture(0, 0))
+ ms_res.stubs(:fetch_hash).returns(*create_thread_group_stats_fixture(min_rate, max_rate))
+ ms_res.stubs(:fetch_hash).returns(nil)
+ mock_mysql_connection.stubs(:query).with(MysqlThreadPoolMonitor::SELECT_THREAD_GROUP_STATS).returns(ms_res)
+ last_run = Time.now
+ plugin_first_run = MysqlThreadPoolMonitor.new(nil, memory = {}, @options)
+ result_one = plugin_first_run.run
+ assert result_one[:reports].length == 0
+ Timecop.travel(last_run + 60) do
+ plugin_second_run = MysqlThreadPoolMonitor.new(last_run, result_one[:memory], @options)
+ result_two = plugin_second_run.run
+ MysqlThreadPoolMonitor::HEADERS_TO_TRACK_RATE_OF.each { |tracked_header|
+ report_key_avg = "#{tracked_header.downcase}_rate_avg"
+ result_two[:reports].each { |report|
+ if report[report_key_avg] then
+ assert_equal 5.0, report[report_key_avg], report_key_avg + " " + report.inspect
+ end
+ }
+ }
+ end
+ end
+
+private
+
+ # Mock out the database connection
+ def mock_mysql_connection
+ fake_mysql = mock
+ Mysql.stubs(:new).returns(fake_mysql)
+ return fake_mysql
+ end
+
+ # Returns an Array of Hashes for the combined result set from the SELECT_THREAD_GROUP_STATS statement
+ def create_thread_group_stats_fixture(increment_min, increment_max)
+ lines = File.readlines("fixture_thread_group_stats.tsv")
+ headers = lines.shift.strip.split("\t")
+ used_inc_min = false
+ used_inc_max = false
+ result_set = []
+ lines.each { |line|
+ if not used_inc_min then
+ increment = increment_min
+ used_inc_min = true
+ elsif not used_inc_max then
+ increment = increment_max
+ used_inc_max = true
+ else
+ increment = ((increment_min.to_f + increment_max.to_f) / 2.0).to_i
+ end
+ result = {}
+ stripped_line = line.strip
+ columns = stripped_line.split("\t")
+ headers.each { |header|
+ result[header] = columns.shift.to_i + increment
+ }
+ result_set << result
+ }
+ result_set
+ end
+
+end

0 comments on commit fba3b60

Please sign in to comment.