Merge pull request #20 from asher/master

Support for centralized review dbs, and a few extra switches

conf.php.sample

@@ -1,9 +1,4 @@
 <?php
-	# I'm guessing that eventually more elaborate configuration will be needed
-	# since the mk-digest db might live on an entirely different host. That's
-	# not the case for us right now, so we'll make that case work later
-	# - Mike 2010-04-04
-
 	# these are default values; they can be overridden on a per-host basis
 	$conf['db_user'] = ''; # user with which to login
 	$conf['db_password'] = ''; # password
@@ -12,6 +7,20 @@
 	# mk-query-digest lets you customize table names
 	$conf['db_query_review_table'] = 'query_review';
 	$conf['db_query_review_history_table'] = 'query_review_history';
+	$conf['limit'] = 20; # max number of slow queries to display, defaults to 20
+	# $conf['explain'] = false; # set to disable explain functionality, default is true
+	# $conf['anon'] = true; # set to only display query fingerprints, default is false
+
+	# You can aggregate the review tables for many db instances in a single db.
+	# In that case, ishmael will auto-generate the list of hosts based on the
+	# contents of the aggregate db. $conf['hosts'] will not be used.
+	#
+	# If you have db{1..20}, use table names like db1_query_review.
+	# define db_central_host and add a wildcard to the table definitions:
+	#
+	# $conf['db_central_host'] = ''; # only define if aggregating many hosts to one db
+	# $conf['db_query_review_table'] = '%query_review';
+	# $conf['db_query_review_history_table'] = '%query_review_history';
 
 	$conf['hosts'] = array(
 		'sample.host' => array(
@@ -20,6 +29,7 @@
 			# 'db_user' => 'foo',
 			# 'db_password' => 'blah',
 			# values available only at the host level:
+			# 'port' => '3308', # optional if using a non-default port
 			# 'label' => 'shard1a' # used for easier host identification
 		)
 	);

dashboard.php

@@ -5,7 +5,9 @@
 
 	require_once('init.php');
 
-	$sort = ($_GET['sort']) ? $_GET['sort'] : "ratio";
+	$sort = ($_GET['sort']) ? mysql_real_escape_string($_GET['sort']) : "ratio";
+	$limit = ($conf['limit']) ? $conf['limit'] : 20;
+	$explain = (isset($conf['explain'])) ? $conf['explain'] : true;
 
 	$aggregate='%m/%d/%y %H:00:00';
 	if ($hours > 720){
@@ -59,25 +61,36 @@
 
 	# Get list of bad queries
 	$q = "SELECT
-			checksum,
-			sample,
-			SUM(ts_cnt) AS count,
-			SUM(query_time_sum) AS time,
-			ts_max AS time_max,
-			(SUM(ts_cnt)/{$query_qty_sum}*100) AS qty_pct,
-			(SUM(query_time_sum)/{$query_time_sum}*100) AS time_pct,
-			((SUM(query_time_sum)/{$query_time_sum}*100)/(SUM(ts_cnt)/{$query_qty_sum}*100)) AS ratio
+			h.checksum,
+			r.fingerprint,
+			h.sample,
+			SUM(h.ts_cnt) AS count,
+			SUM(h.query_time_sum) AS time,
+			h.ts_max AS time_max,
+			(SUM(h.ts_cnt)/{$query_qty_sum}*100) AS qty_pct,
+			(SUM(h.query_time_sum)/{$query_time_sum}*100) AS time_pct,
+			((SUM(h.query_time_sum)/{$query_time_sum}*100)/(SUM(h.ts_cnt)/{$query_qty_sum}*100)) AS ratio
 		FROM 
-			{$host_conf['db_query_review_history_table']}
-		WHERE 
-			ts_max > date_sub(now(),interval $hours hour) 
-		GROUP BY checksum ORDER BY $sort DESC LIMIT 20";
+			{$host_conf['db_query_review_history_table']} h
+		INNER JOIN
+			{$host_conf['db_query_review_table']} r
+		USING(checksum) WHERE 
+			h.ts_max > date_sub(now(),interval $hours hour) 
+		GROUP BY h.checksum ORDER BY $sort DESC LIMIT $limit";
 
 	$result = mysql_query($q);
 	$err = mysql_error();
 	print_r($err);
 	$rows = array();
 	while ($row = mysql_fetch_assoc($result)) {
+		if ($conf['anon']) {
+			if (preg_match('@/\* (\S+)(::\S+)?.*\*/@', $row['sample'], $matches)) {
+				$c = ($matches[2]) ? $matches[1] . $matches[2] : $matches[1];
+				$row['sample'] = '/* ' . $c . ' */ ' . $row['fingerprint'];
+			} else {
+				$row['sample'] = $row['fingerprint'];
+			}
+		}
 		$row['explain_url'] = "explain.php?" . ish_build_query(array('checksum'=>$row['checksum']));
 		$row['more_url'] = "more.php?" . ish_build_query(array('checksum'=>$row['checksum']));
 		$rows[] = $row;

dashboard.tpl

@@ -58,9 +58,11 @@
 				<td><? printf("%.2f", $row['qty_pct']) ?> <br /> <small>(<?= $row['count'] ?>)</small></td>
 				<td><? printf("%.2f", $row['ratio']) ?></td>
 				<td><?=format_query($row['sample']) ?></td>
+				<? if ($explain): ?>
 				<td><a href="<?= $row['explain_url'] ?>">explain</a></td>
+				<? endif ?>
 				<td><a href="<?= $row['more_url'] ?>">more</a></td>
 			</tr>
 		<? endforeach ?>
 		</table>
-<?php include("bottom.tpl"); ?>
+<?php include("bottom.tpl"); ?>

explain.php

@@ -4,16 +4,24 @@
 	#
 
 	require_once('init.php');
+	$explain = (isset($conf['explain'])) ? $conf['explain'] : true;
+	$anon = (isset($conf['anon'])) ? $conf['anon'] : false;
+	if (!$explain || $anon) {
+		header("HTTP/1.0 404 Not Found");
+		header("Status: 404 Not Found");
+		return;
+	}
 
-	$checksum = $_GET['checksum'];
-	$hours = $_GET['hours'];
+	$checksum = mysql_real_escape_string($_GET['checksum']);
+	$hours = mysql_real_escape_string($_GET['hours']);
 
 	$q = "SELECT * FROM {$host_conf['db_query_review_history_table']} WHERE checksum = '{$checksum}'";
 	$result = mysql_query($q);
 	$row = mysql_fetch_assoc($result);
 	$query = $row['sample'];
 	$explain = "EXPLAIN ".preg_replace('|^\s*\/\*.*\*\/|', '', $query);
 
+	mysql_connect($conf['db_live_host'],$conf['db_live_user'],$conf['db_live_password']) or die(sprintf("Unable to connect to MySQL server: %s", mysql_error()));
 	mysql_select_db($host_conf['db_database_live']) or die("Unable to select database");
 
 	$is_select_query = (stripos($explain, 'SELECT') !== false);

init.php

@@ -42,27 +42,67 @@ function format_query($query) {
 		return "<span class=\"query\">{$query}</span>";
 	}
 
+	function ish_get_central_host_list() {
+		global $conf;
+		$hosts = array();
+		mysql_connect($conf['db_central_host'],$conf['db_user'],$conf['db_password']) or die(sprintf("Unable to connect to MySQL server: %s", mysql_error()));
+		mysql_select_db($conf['db_database_mk']) or die(sprintf("Unable to select database: %s", mysql_error()));
+		# get list of host review tables
+		$q = 'SHOW TABLES LIKE "' . $conf['db_query_review_table'] . '"';
+		$result = mysql_query($q);
+		$rows = array();
+		while ($row = mysql_fetch_row($result)) {
+			preg_match('/^([a-z0-9.-]+)_.*/', $row[0], $match);
+			if ($match[1]) {
+				$host = $match[1];
+				$hosts[$host] = $host;
+			}
+		}
+		return $hosts;
+	}
+
+	function ish_get_central_host_config($host) {
+		global $conf;
+		$host_config = $conf;
+		$host_config['db_host'] = $conf['db_central_host'];
+		$host_config['title'] = $host;
+		$host_config['label'] = $host;
+		$host_config['db_query_review_table'] = preg_replace('/%/', $host.'_', $conf['db_query_review_table']);
+		$host_config['db_query_review_history_table'] = preg_replace('/%/', $host.'_', $conf['db_query_review_history_table']);
+		return($host_config);
+	}
+
 	# get the list of hosts this ishmael install is configured to look at
 	function ish_get_host_list() {
 		global $conf;
 		$host_list = array();
+		if ($conf['db_central_host']) {
+			return ish_get_central_host_list();
+		}
 		foreach (array_keys($conf['hosts']) as $host) {
 			$host_config = ish_get_host_config($host);
 			$host_list[$host] = $host_config['title'];
 		}
 		return $host_list;
 	}
 
+
 	# merges the config for a particular host on top of the defaults
 	function ish_get_host_config($host) {
 		global $conf;
+		if ($conf['db_central_host']) {
+			return ish_get_central_host_config($host);
+		}
 		$defaults = $conf;
 		unset($defaults['hosts']);
 		$defaults['db_host'] = $host;
 		$host_config = array_merge($defaults, $conf['hosts'][$host]);
 		$host_config['title'] = $host_config['label']
 			? "{$host_config['label']} - {$host_config['db_host']}"
 			: $host_config['db_host'];
+		if (array_key_exists('port', $host_config)) {
+			$host_config['db_host'] .= ':' . $host_config['port'];
+		}
 		return $host_config;
 	}
 
@@ -82,11 +122,11 @@ function ish_build_query($args) {
 	$hosts = ish_get_host_list();
 
 	# which host are we looking at
-	$host = $_GET['host'] ? $_GET['host'] : reset(array_keys($hosts));
+	$host = $_GET['host'] ? mysql_real_escape_string($_GET['host']) : reset(array_keys($hosts));
 	$host_conf = ish_get_host_config($host);
 
 	# what timeframe we want to look at
-	$hours = $_GET['hours'] ? $_GET['hours'] : 24;
+	$hours = $_GET['hours'] ? mysql_real_escape_string($_GET['hours']) : 24;
 
 	mysql_connect($host_conf['db_host'],$host_conf['db_user'],$host_conf['db_password']) or die(sprintf("Unable to connect to MySQL server: %s", mysql_error()));
 	mysql_select_db($host_conf['db_database_mk']) or die(sprintf("Unable to select database: %s", mysql_error()));

more.php

@@ -5,7 +5,7 @@
 
 	require_once('init.php');
 
-	$checksum = $_GET['checksum'];
+	$checksum = mysql_real_escape_string($_GET['checksum']);
 
 	$fields = array(
 		'Query_time',
@@ -19,17 +19,27 @@
 	);
 
 	$q = "SELECT 
-			*
+			h.*, r.fingerprint
 		FROM
-			{$host_conf['db_query_review_history_table']}
-		WHERE
+			{$host_conf['db_query_review_history_table']} h
+		INNER JOIN
+			{$host_conf['db_query_review_table']} r
+		USING(checksum) WHERE
 			checksum={$checksum} AND
 			ts_max > date_sub(now(),interval $hours hour) 
 		ORDER BY ts_max ASC";
 	$result=mysql_query($q);
 
 	$rows = array();
 	while ($row = mysql_fetch_assoc($result)) {
+		if ($conf['anon']) {
+			if (preg_match('@/\* (\S+)(::\S+)?.*\*/@', $row['sample'], $matches)) {
+				$c = ($matches[2]) ? $matches[1] . $matches[2] : $matches[1];
+				$row['sample'] = '/* ' . $c . ' */ ' . $row['fingerprint'];
+			} else {
+				$row['sample'] = $row['fingerprint'];
+			}
+		}
 		$rows[] = $row;
 	}
 

table.php

@@ -5,8 +5,8 @@
 
         require_once('init.php');
 
-        $table = $_GET['table'];
-	$hours = $_GET['hours'];
+        $table = mysql_real_escape_string($_GET['table']);
+	$hours = mysql_real_escape_string($_GET['hours']);
 
         $q = "SELECT
                         t.sample as query, sum(h.ts_cnt) as count, t.checksum as checksum