Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding Percona support #128

Merged
merged 2 commits into from
May 10, 2016
Merged

Adding Percona support #128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
42b844e
Adding Percona support
Apr 17, 2016
eba10b7
Adding Percona support
Apr 17, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 76 additions & 16 deletions modules/mysql.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,34 +6,43 @@
local l = require "lpeg"
l.locale(l)
local tonumber = tonumber
local ip = require "ip_address"

local M = {}
setfenv(1, M) -- Remove external access to contain everything in the module

local pct_encoded = l.P"%" * l.xdigit * l.xdigit
local unreserved = l.alnum + l.S"-._~"
local sub_delims = l.S"!$&'()*+,;="

local space = l.space^1
local sep = l.P"\n"
local sql_end = l.P";" * (l.P"\n" + -1)
local line = (l.P(1) - sep)^0 * sep
local float = l.digit^1 * "." * l.digit^1
local float = l.digit^1 * "." * l.digit^1 / tonumber
local integer = l.P"-"^-1 * l.digit^1 / tonumber
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't see any negative counts in the spec have you seen them in the actual output?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe that I've seen them previously, but I can't seem to find any concrete examples to share at the moment.

local host = l.Ct(l.Cg(ip.v4, "value") * l.Cg(l.Cc"ipv4", "representation"))
+ l.Ct(l.Cg(ip.v6, "value") * l.Cg(l.Cc"ipv6", "representation"))
+ l.Ct(l.Cg((unreserved + pct_encoded + sub_delims)^1, "value") * l.Cg(l.Cc"hostname", "representation"))
Copy link
Contributor

@trink trink Apr 25, 2016 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We now have something like this is multiple places. We should probably pull host/hostname out into a common grammar module like ip_address.lua (not necessary for this PR)

#129

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this remain for now?


local time = l.P"# Time: " * line

local user_legal = (1 - l.S"[]")
local host_legal = (l.alnum + l.S".-")
local user_name = user_legal^1 * "[" * l.Cg(user_legal^1, "Username") * "]"
local user_name = user_legal^0 * "[" * l.Cg(user_legal^0, "Username") * "]"
Copy link
Contributor

@trink trink Apr 25, 2016 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will need to double check to see if user_name can validly be empty.

Edit: yes it can, so this is fine

local host_name = host_legal^0 * l.space^0 * "[" * l.Cg((l.P(1) - "]")^1, "Hostname") * "]"
local user = l.P"# User@Host: " * user_name * space * "@" * space * host_name * sep

local query_time = l.P"# Query_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") * l.Cg(l.Cc"s", "representation")), "Query_time")
local lock_time = l.P"Lock_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") * l.Cg(l.Cc"s", "representation")), "Lock_time")
local rows_sent = l.P"Rows_sent: " * l.Cg(l.digit^1 / tonumber, "Rows_sent")
local rows_examined = l.P"Rows_examined: " * l.Cg(l.digit^1 / tonumber, "Rows_examined")
local query_time = l.P"# Query_time: " * l.Cg(l.Ct(l.Cg(float, "value") * l.Cg(l.Cc"s", "representation")), "Query_time")
local lock_time = l.P"Lock_time: " * l.Cg(l.Ct(l.Cg(float, "value") * l.Cg(l.Cc"s", "representation")), "Lock_time")
local rows_sent = l.P"Rows_sent: " * l.Cg(integer, "Rows_sent")
local rows_examined = l.P"Rows_examined: " * l.Cg(integer, "Rows_examined")
local query = query_time * space * lock_time * space * rows_sent * space * rows_examined * sep

local use_db = l.P"use " * line

local last_insert = l.P"last_insert_id=" * l.digit^1 * ","
local insert = l.P"insert_id=" * l.digit^1 * ","
local last_insert = l.P"last_insert_id=" * l.Cg(integer, "Last_insert") * ","
local insert = l.P"insert_id=" * l.Cg(integer, "Insert_id") * ","
local timestamp = l.P"timestamp=" * l.Cg((l.digit^1 / "%0000000000"), "Timestamp")
local set = l.P"SET " * last_insert^0 * insert^0 * timestamp * ";" * sep

Expand All @@ -43,22 +52,73 @@ local sql = l.Cg((l.P(1) - sql_end)^0 * sql_end, "Payload")

-- Maria DB extensions
local yes_no = l.C(l.P"Yes" + "No")
local thread_id = l.P"# Thread_id: " * l.Cg(l.digit^1 / tonumber, "Thread_id")
* l.P" Schema: " * l.Cg(l.alnum^0, "Schema")
local thread_id = l.P"# Thread_id: " * l.Cg(integer, "Thread_id")
* l.P" Schema: " * l.Cg(unreserved^0, "Schema")
* l.P" QC_hit: " * l.Cg(yes_no, "QC_hit") * sep

local full_scan = l.P"# Full_scan: " * l.Cg(yes_no, "Full_scan")
* l.P" Full_join: " * l.Cg(yes_no, "Full_join")
* l.P" Tmp_table: " * l.Cg(yes_no, "Tmp_table")
* l.P" Tmp_table_on_disk: " * l.Cg(yes_no, "Tmp_table_on_disk") * sep
* " Full_join: " * l.Cg(yes_no, "Full_join")
* " Tmp_table: " * l.Cg(yes_no, "Tmp_table")
* " Tmp_table_on_disk: " * l.Cg(yes_no, "Tmp_table_on_disk") * sep
* l.P"# Filesort: " * l.Cg(yes_no, "Filesort")
* " Filesort_on_disk: " * l.Cg(yes_no, "Filesort_on_disk")
* " Merge_passes: " * l.Cg(l.digit^1 / tonumber, "Merge_passes") * sep
* " Merge_passes: " * l.Cg(integer, "Merge_passes") * sep

-- Percona extensions
local percona_user = "# User@Host: " * user_name * space * "@" * space * host_name * sep^0
local conn_id = " Id:" * space * l.Cg(integer, "Connection_id") * sep

local info = l.P(("# Thread_id: " * l.Cg(integer, "Thread_id")
* " Schema: " * l.Cg(unreserved^0, "Schema")
* " Last_errno: " * l.Cg(integer, "Last_errno")
* " Killed: " * l.Cg(integer, "Killed"))
+ ("# Schema: " * l.Cg(unreserved^0, "Schema")
* " Last_errno: " * l.Cg(integer, "Last_errno")
* " Killed: " * l.Cg(integer, "Killed"))) * sep

local percona_query = "# Query_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value")
* l.Cg(l.Cc"s", "representation")), "Query_time")
* " Lock_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value")
* l.Cg(l.Cc"s", "representation")), "Lock_time")
* " Rows_sent: " * l.Cg(integer, "Rows_sent")
* " Rows_examined: " * l.Cg(integer, "Rows_examined")
* " Rows_affected: " * l.Cg(integer, "Rows_affected")
* (" Rows_read: " * l.Cg(integer, "Rows_read"))^0 * sep

local memory_footprint = "# Bytes_sent: " * l.Cg(l.Ct(l.Cg(integer, "value")
* l.Cg(l.Cc"B", "representation")), "Bytes_sent")
* (" Tmp_tables: " * l.Cg(integer, "Tmp_tables")
* " Tmp_disk_tables: " * l.Cg(integer, "Tmp_disk_tables")
* " Tmp_table_sizes: " * l.Cg(l.Ct(l.Cg(integer, "value")
* l.Cg(l.Cc"B", "representation")), "Tmp_table_sizes"))^0 * sep

local stored_routine = "# Stored routine: " * l.Cg((l.alnum + l.S"_.-")^1, "Stored_routine") * sep

local query_plan_info = "# QC_Hit: " * l.Cg(yes_no, "QC_hit")
* " Full_scan: " * l.Cg(yes_no, "Full_scan")
* " Full_join: " * l.Cg(yes_no, "Full_join")
* " Tmp_table: " * l.Cg(yes_no, "Tmp_table")
* " Tmp_table_on_disk: " * l.Cg(yes_no, "Tmp_table_on_disk") * sep
* "# Filesort: " * l.Cg(yes_no, "Filesort")
* " Filesort_on_disk: " * l.Cg(yes_no, "Filesort_on_disk")
* " Merge_passes: " * l.Cg(integer, "Merge_passes") * sep

local innodb_usage = "# InnoDB_IO_r_ops: " * l.Cg(integer, "InnoDB_IO_r_ops")
* " InnoDB_IO_r_bytes: " * l.Cg(l.Ct(l.Cg(integer, "value")
* l.Cg(l.Cc"B", "representation")), "InnoDB_IO_r_bytes")
* " InnoDB_IO_r_wait: " * l.Cg(l.Ct(l.Cg(float, "value")
* l.Cg(l.Cc"s", "representation")), "InnoDB_IO_r_wait") * sep
* "# InnoDB_rec_lock_wait: " * l.Cg(l.Ct(l.Cg(float, "value")
* l.Cg(l.Cc"s", "representation")), "InnoDB_rec_lock_wait")
* " InnoDB_queue_wait: " * l.Cg(l.Ct(l.Cg(float, "value")
* l.Cg(l.Cc"s", "representation")), "InnoDB_queue_wait") * sep
* "# InnoDB_pages_distinct: " * l.Cg(integer, "InnoDB_pages_distinct") * sep

slow_query_grammar = l.Ct(time^0 * user * l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql)
mariadb_slow_query_grammar = l.Ct(time^0 * user * l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql)
percona_slow_query_grammar = l.Ct(time^0 * percona_user * l.Cg(l.Ct(conn_id^0 * info^0 * percona_query * memory_footprint^0 * stored_routine^0 * query_plan_info^0 * innodb_usage^0), "Fields") * use_db^0 * set * admin^0 * sql)

short_slow_query_grammar = l.Ct(l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql)
mariadb_short_slow_query_grammar= l.Ct(l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql)
short_slow_query_grammar = l.Ct(l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql)
mariadb_short_slow_query_grammar = l.Ct(l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql)

return M
112 changes: 105 additions & 7 deletions src/test/lua/lpeg_mysql.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,42 @@ WHERE id = 10;
]]
}

local percona_slow_query_log = {
[[
# User@Host: syncrw[syncrw] @ db-01.example.com [127.0.0.1] Id: 42
# Schema: imdb Last_errno: 0 Killed: 0
# Query_time: 7.725616 Lock_time: 0.000328 Rows_sent: 4 Rows_examined: 1543720 Rows_affected: 0
# Bytes_sent: 272 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0
# QC_Hit: No Full_scan: Yes Full_join: No Tmp_table: No Tmp_table_on_disk: No
# Filesort: No Filesort_on_disk: No Merge_passes: 0
SET timestamp=1399500744;
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;
]],
[[
# Time: 130601 8:01:06
# User@Host: syncrw[syncrw] @ db-01.example.com [127.0.0.1] Id: 42
# Schema: imdb Last_errno: 0 Killed: 0
# Query_time: 7.725616 Lock_time: 0.000328 Rows_sent: 4 Rows_examined: 1543720 Rows_affected: 0 Rows_read: 30
# Bytes_sent: 272 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0
# QC_Hit: No Full_scan: Yes Full_join: No Tmp_table: No Tmp_table_on_disk: No
# Filesort: No Filesort_on_disk: No Merge_passes: 0
# InnoDB_IO_r_ops: 6415 InnoDB_IO_r_bytes: 105103360 InnoDB_IO_r_wait: 0.001279
# InnoDB_rec_lock_wait: 0.000000 InnoDB_queue_wait: 0.000000
# InnoDB_pages_distinct: 6430
SET timestamp=1399500744;
/* [queryName=FIND_ITEMS] */ SELECT *
FROM widget
WHERE id = 10;
]]
}

local fields = {
{"Query_time", 2.964652, "s"},
{"Rows_examined", 9773},
{"Lock_time", 0.000050, "s"},
{"Rows_sent", 251},
{"Rows_sent", 251}
}

local mariadb_fields = {
Expand Down Expand Up @@ -118,6 +149,59 @@ local mariadb_verbose_fields = {
{"Merge_passes", 3}
}

local percona_fields = {
{"Schema", "imdb"},
{"Last_errno", 0},
{"Killed", 0},
{"Query_time", 7.725616, "s"},
{"Lock_time", 0.000328, "s"},
{"Rows_sent", 4},
{"Rows_examined", 1543720},
{"Rows_affected", 0},
{"Bytes_sent", 272, "B"},
{"Tmp_tables", 0},
{"Tmp_disk_tables", 0},
{"Tmp_table_sizes", 0, "B"},
{"QC_hit", "No"},
{"Full_scan", "Yes"},
{"Full_join", "No"},
{"Tmp_table", "No"},
{"Tmp_table_on_disk", "No"},
{"Filesort", "No"},
{"Filesort_on_disk", "No"},
{"Merge_passes", 0}
}

local percona_verbose_fields = {
{"Schema", "imdb"},
{"Last_errno", 0},
{"Killed", 0},
{"Query_time", 7.725616, "s"},
{"Lock_time", 0.000328, "s"},
{"Rows_sent", 4},
{"Rows_examined", 1543720},
{"Rows_affected", 0},
{"Rows_read", 30},
{"Bytes_sent", 272, "B"},
{"Tmp_tables", 0},
{"Tmp_disk_tables", 0},
{"Tmp_table_sizes", 0, "B"},
{"QC_hit", "No"},
{"Full_scan", "Yes"},
{"Full_join", "No"},
{"Tmp_table", "No"},
{"Tmp_table_on_disk", "No"},
{"Filesort", "No"},
{"Filesort_on_disk", "No"},
{"Merge_passes", 0},
{"InnoDB_IO_r_ops", 6415},
{"InnoDB_IO_r_bytes", 105103360, "B"},
{"InnoDB_IO_r_wait", 0.001279, "s"},
{"InnoDB_rec_lock_wait", 0, "s"},
{"InnoDB_queue_wait", 0, "s"},
{"InnoDB_pages_distinct", 6430}
}

local function validate(fields, t)
if t.Timestamp ~= "1399500744000000000" then return error("Timestamp:" .. t.Timestamp) end
if t.Payload ~= "/* [queryName=FIND_ITEMS] */ SELECT *\nFROM widget\nWHERE id = 10;\n" then return error("Payload:" .. t.Payload) end
Expand All @@ -131,6 +215,7 @@ local function validate(fields, t)
end
end
else
print("Output:" .. v[1])
if t.Fields[v[1]] ~= v[2] then return error(string.format("field:%s:", v[1]) .. t.Fields[v[1]], 2) end
end
end
Expand All @@ -139,45 +224,56 @@ end
local function header()
local t = mysql.slow_query_grammar:match(slow_query_log[1])
if not t then return error("no match") end
if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end
if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end
validate(fields, t)
end

local function standard()
local t = mysql.slow_query_grammar:match(slow_query_log[2])
if not t then return error("no match") end
if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end
if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end
validate(fields, t)
end

local function short()
local t = mysql.short_slow_query_grammar:match(slow_query_log[3])
if not t then return error("no match") end
if t.Hostname then return error("Hostname:" .. t.Hostname) end
validate(fields, t)
end

local function mariadb_standard()
local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[1])
if not t then return error("no match") end
if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end
if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end
validate(mariadb_fields, t)
end

local function mariadb_short()
local t = mysql.mariadb_short_slow_query_grammar:match(mariadb_slow_query_log[2])
if not t then return error("no match") end
if t.Hostname then return error("Hostname:" .. t.Hostname) end
validate(mariadb_fields, t)
end

local function mariadb_verbose()
local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[3])
if not t then return error("no match") end
if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end
if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end
validate(mariadb_verbose_fields, t)
end

local function percona_standard()
local t = mysql.percona_slow_query_grammar:match(percona_slow_query_log[1])
if not t then return error("no match") end
if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end
validate(percona_fields, t)
end

local function percona_verbose()
local t = mysql.percona_slow_query_grammar:match(percona_slow_query_log[2])
if not t then return error("no match") end
if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end
validate(percona_verbose_fields, t)
end

function process(tc)
header()
Expand All @@ -186,6 +282,8 @@ function process(tc)
mariadb_standard()
mariadb_short()
mariadb_verbose()
percona_standard()
percona_verbose()

return 0
end