sql perf graphs and live grafana/influxdb health checks

eostermueller · Dec 5, 2015 · cf0aa09 · cf0aa09
1 parent 937156f
commit cf0aa09
Show file tree

Hide file tree

Showing 88 changed files with 3,127 additions and 866 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@ export-dir/
 datatables/target/
 wuqiSpank/.extract/
 wuqiSpank/target
+wuqiSpank/bin/logs/
 *.class
 export-dir/
 wuqiSpank/.classpath

diff --git a/wuqiSpank/bin/startup.sh b/wuqiSpank/bin/startup.sh
@@ -9,4 +9,4 @@ else
 fi
 
 echo Writing to wuqiSpank logfile $(dirname $0)/logs/wuqiSpank-$$.log
-java -jar wuqiSpank.jar -httpPort 8071 1> $(dirname $0)/logs/wuqiSpank-$$.log 2>&1 &
+java -Xmx3g -jar wuqiSpank.jar  1> $(dirname $0)/logs/wuqiSpank-$$.log 2>&1 &
diff --git a/wuqiSpank/pom.xml b/wuqiSpank/pom.xml
@@ -4,7 +4,8 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 
   <properties>
-	<wicket.version>6.12.0</wicket.version>
+	<!-- <wicket.version>6.12.0</wicket.version>  -->
+	<wicket.version>7.0.0</wicket.version>
   </properties>
   <modelVersion>4.0.0</modelVersion>
   <parent>
@@ -18,7 +19,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
   <name>DataTables jQuery Plugin Integration - Examples</name>
   <developers>
     <developer>
-      <id>brunoborges</id>
+      <id>eostermueller</id>
       <name>Erik Ostermueller</name>
       <email>eostermueller at gmail.com</email>
     </developer>
@@ -92,6 +93,43 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
 
 
   <dependencies>
+	<dependency>
+            <groupId>org.influxdb</groupId>
+            <artifactId>influxdb-java</artifactId>
+            <version>2.0-SNAPSHOT</version>
+        </dependency>
+
+        <!-- Start of influxdb dependencies -->
+              <dependency>
+                        <groupId>org.testng</groupId>
+                        <artifactId>testng</artifactId>
+                        <version>6.9.4</version>
+                        <scope>test</scope>
+                </dependency>
+                <dependency>
+                        <groupId>com.google.guava</groupId>
+                        <artifactId>guava</artifactId>
+                        <version>18.0</version>
+                </dependency>
+                <dependency>
+                        <groupId>com.squareup.retrofit</groupId>
+                        <artifactId>retrofit</artifactId>
+                        <version>1.9.0</version>
+                </dependency>
+                <!-- If we use okhttp instead of java urlconnection we achieve server failover
+                        of the influxdb server address resolves to all influxdb server ips. -->
+                <dependency>
+                        <groupId>com.squareup.okhttp</groupId>
+                        <artifactId>okhttp</artifactId>
+                        <version>2.4.0</version>
+                </dependency>
+          		<dependency>
+                        <groupId>org.testng</groupId>
+                        <artifactId>testng</artifactId>
+                        <version>6.9.4</version>
+                        <scope>test</scope>
+                </dependency>                        
+        <!--  End of influxdb dependencies -->  
 	<dependency>
 		<groupId>com.lowagie</groupId>
 		<artifactId>itext</artifactId>
@@ -103,6 +141,19 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
       <artifactId>graph-lib</artifactId>
       <version>2.4.0.4</version>
     </dependency>
+    <dependency>
+    <!-- Metrics helps provide live activity reports over various time intervals. 
+    	last 10 seconds, last minute, last hour, day, etc. -->
+        <groupId>io.dropwizard.metrics</groupId>
+        <artifactId>metrics-core</artifactId>
+        <version>3.1.2</version>
+    </dependency>
+	<dependency>
+		<groupId>io.dropwizard.metrics</groupId>
+		<artifactId>metrics-healthchecks</artifactId>
+		<version>3.1.2</version>
+	</dependency>
+
     <!-- The following is a transitive dependency to headlessInTraceClient-1.0.jar -->
     <dependency>
       <groupId>net.java.dev.glazedlists</groupId>
@@ -112,8 +163,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
     <dependency>
       <groupId>org.wicketstuff</groupId>
       <artifactId>wicketstuff-datatables</artifactId>
-      <version>6.11.0</version>
-      <!--                    <version>${wicket.version}</version>  -->
+      <!--  <version>6.11.0</version>  -->
+      <version>${wicket.version}</version>
     </dependency>
     <dependency>
       <groupId>com.github.eostermueller</groupId>

diff --git a/wuqiSpank/src/main/java/org/wicketstuff/datatables/JQueryDatatable.java b/wuqiSpank/src/main/java/org/wicketstuff/datatables/JQueryDatatable.java
@@ -40,6 +40,7 @@ public void renderHead(IHeaderResponse response)
 		js.append("	var oTable = $(\"#" + getMarkupId() + "\").dataTable( {\n");
 		//js.append("		\"bJQueryUI\": true,\n");
 		js.append( horizontalScrollConfiguration() );
+		js.append("\"autoWidth\": true");
 		js.append( columnConfiguration() );
 //		js.append("		\"sPaginationType\": \"full_numbers\"\n");
 		js.append("	} );\n");

diff --git a/wuqiSpank/src/main/java/org/wuqispank/DefaultFactory.java b/wuqiSpank/src/main/java/org/wuqispank/DefaultFactory.java
@@ -1,11 +1,18 @@
 package org.wuqispank;
 
+import java.util.SortedMap;
+import java.util.TreeMap;
+
 import javax.xml.parsers.ParserConfigurationException;
 
 import org.headlessintrace.jdbc.IJdbcProvider;
 import org.wuqispank.db.ISqlParser;
 import org.wuqispank.db.akiban.AkibanSqlParser;
 import org.wuqispank.db.jsqlparser.JSqlParser;
+import org.wuqispank.health.DefaultHealthChecker;
+import org.wuqispank.health.DefaultInTraceHealthCheck;
+import org.wuqispank.health.DefaultTcpHealthCheck;
+import org.wuqispank.health.Result;
 import org.wuqispank.importexport.DefaultExportDirListener;
 import org.wuqispank.importexport.DefaultImportExportMgr;
 import org.wuqispank.importexport.DefaultInTraceEventFileImporter;
@@ -22,6 +29,7 @@
 import org.wuqispank.model.DefaultBinaryOperatorExpression;
 import org.wuqispank.model.DefaultColumn;
 import org.wuqispank.model.DefaultModelObservationMgr;
+import org.wuqispank.model.DefaultRequestManager;
 import org.wuqispank.model.DefaultRequestWrapper;
 import org.wuqispank.model.DefaultSqlModel;
 import org.wuqispank.model.DefaultSqlStatsObserver;
@@ -31,6 +39,7 @@
 import org.wuqispank.model.IBinaryOperatorExpression;
 import org.wuqispank.model.IColumn;
 import org.wuqispank.model.IModelObservationMgr;
+import org.wuqispank.model.IRequestManager;
 import org.wuqispank.model.IRequestRepository;
 import org.wuqispank.model.IRequestWrapper;
 import org.wuqispank.model.ISqlModel;
@@ -52,6 +61,11 @@
 import org.wuqispank.web.tableaccesstimeline.IRowGroup;
 import org.wuqispank.web.tableaccesstimeline.ITableLaneMgr;
 
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.health.HealthCheck;
+import com.codahale.metrics.health.HealthCheckRegistry;
+
+
 
 /**
  * This is my low-complexity approach to dependency injection.
@@ -78,12 +92,17 @@ public ITraceEventParser getEventParser() {
  */
 
 public class DefaultFactory implements IFactory {
+	private static final HealthCheckRegistry healthChecks = new HealthCheckRegistry();
+	private static final MetricRegistry metrics = new MetricRegistry();
 	public static final String RESEARCH_EYE_CATCHER = "@#WUQISPANK_RESEARCH#@:";
-	private static IMessages msgs = new AmericanEnglishMessages();
+	private static final  IMessages msgs = new AmericanEnglishMessages();
 	private static IConfig m_config = null;
-	private static IFactory INSTANCE = new DefaultFactory();
+	private static  IFactory INSTANCE = new DefaultFactory();
 	private static IImportExportMgr m_importExportMgr = new DefaultImportExportMgr();
+	private static final  DefaultRequestManager requestManager = new DefaultRequestManager();
 	private IJdbcProvider m_jdbcProvider;
+	private SortedMap<String, Result> healthCheckResults = null;
+	private Object healthCheckResults_lock = new Object();
 
 	@Override
 	public IRequestWrapper getRequestWrapper() {
@@ -184,7 +203,9 @@ public ISqlStatsObserver getSqlStatsCounter() {
 	}
 	@Override
 	public IRequestRepository createRepo() {
-		return new InMemoryRequstRepo();
+		int capacity = this.getConfig().getCircularBufferSize();
+		int numToDelete = this.getConfig().getNumberOfRequestsToRemoveAtOnce();
+		return new InMemoryRequstRepo(capacity,numToDelete);
 	}
 	@Override
 	public IRequestExporter getRequestExporter() throws ParserConfigurationException {
@@ -272,4 +293,47 @@ public void setImportExportManager(IImportExportMgr val) {
 	public ISqlParser getSecondarySqlParser() {
 		return new JSqlParser();
 	}
+	@Override
+	public IRequestManager getRequestManager() {
+		return requestManager;
+	}
+	@Override
+	public MetricRegistry getMetricRegistry() {
+		return DefaultFactory.metrics;
+	}
+	@Override
+	public HealthCheckRegistry getHealthCheckRegistry() {
+		return DefaultFactory.healthChecks;
+	}
+
+	/**
+	 * From one thread, DefaultHealthChecker will be collecting results from processes with quesitonable health.
+	 * From a different thread, browser/apache wicket clients will be retrieving this to update wuqiSpank UI.
+	 */
+	@Override
+	public SortedMap<String, org.wuqispank.health.Result> getHealthCheckResults() {
+		SortedMap<String, org.wuqispank.health.Result> copyOfResults = new TreeMap<String, org.wuqispank.health.Result>();
+		synchronized (this.healthCheckResults_lock) {
+			copyOfResults = DefaultHealthChecker.cloneHealthCheckResults(this.healthCheckResults);
+		}
+		return copyOfResults;
+	}
+	@Override
+	public void setHealthCheckResults(SortedMap<String, Result> val) {
+		synchronized (this.healthCheckResults_lock) {
+			this.healthCheckResults = val;
+		}
+	}
+	@Override
+	public Runnable getHealthChecker(HealthCheckRegistry registry) {
+		return new DefaultHealthChecker(registry);
+	}
+	@Override
+	public HealthCheck getInTraceHealthCheck() {
+		return new DefaultInTraceHealthCheck();
+	}
+	@Override
+	public DefaultTcpHealthCheck getTcpHealthCheck() {
+		return new DefaultTcpHealthCheck();
+	}
 }
diff --git a/wuqiSpank/src/main/java/org/wuqispank/DefaultReconnector.java b/wuqiSpank/src/main/java/org/wuqispank/DefaultReconnector.java
@@ -13,10 +13,13 @@
 public class DefaultReconnector implements IReconnector {
 	static Logger LOG = LoggerFactory.getLogger(DefaultReconnector.class);
 
+	/**
+	 * Recreates a connection (a socket) with Headless Intrace, but 
+	 * only if the connection has been broken.
+	 */
 	@Override
 	public void run() {
 		for(IConnection con : DefaultConnectionList.getSingleton().getConnections()) {
-
 			if (con instanceof RequestConnection) {
 				RequestConnection reqCon = (RequestConnection)con;
 				if (!reqCon.isConnected()) {
@@ -25,8 +28,19 @@ public void run() {
 				}
 			}
 		}
+	}
+	@Override
+	public void disconnectAll() {
+		for(IConnection con : DefaultConnectionList.getSingleton().getConnections()) {
+			if (con instanceof RequestConnection) {
+				RequestConnection reqCon = (RequestConnection)con;
+				if (reqCon.isConnected()) {
+					reqCon.disconnect();
+					LOG.debug("Reconnected? [" + reqCon.isConnected() + "] [" + con.getHostPort().toString3() + "] ");
+				}
+			}
+		}
 
-
 	}
 
 	private void connect(RequestConnection reqCon) {
@@ -45,5 +59,22 @@ private void connect(RequestConnection reqCon) {
 		} 
 
 	}
+	@Override
+	public IConnection getConnection() throws WuqispankException {
+		IConnection singleConn = null;
+		int connectionCount = 0;
+
+		for(IConnection con : DefaultConnectionList.getSingleton().getConnections()) {
+			connectionCount++;
+			if (con instanceof RequestConnection) {
+				singleConn = con;
+			}
+		}
+		if (connectionCount !=1) {
+			throw new WuqispankException("Was expecting exactly 1 connection to intrace-agent.jar, but instead found [" + connectionCount + "]");
+		}
+		return singleConn;
+
+	}
 
 }
diff --git a/wuqiSpank/src/main/java/org/wuqispank/IReconnector.java b/wuqiSpank/src/main/java/org/wuqispank/IReconnector.java
@@ -1,5 +1,11 @@
 package org.wuqispank;
 
+import org.headlessintrace.client.connection.IConnection;
+
 public interface IReconnector extends Runnable {
 
+	void disconnectAll();
+
+	IConnection getConnection() throws WuqispankException;
+
 }
diff --git a/wuqiSpank/src/main/java/org/wuqispank/db/akiban/AkibanSqlParser.java b/wuqiSpank/src/main/java/org/wuqispank/db/akiban/AkibanSqlParser.java
@@ -51,6 +51,10 @@ public class AkibanSqlParser implements ISqlParser {
 	@SuppressWarnings("static-access")
 	@Override
 	public void parse(String sqlText) throws SqlParseException {
+		if (this.getSqlModel()==null) {
+			throw new SqlParseException(DefaultFactory.getFactory().getMessages().missingSqlModel(sqlText));
+		}
+		this.getSqlModel().setParser(this.getClass());
 		init();
 		if (sqlText==null || "".equals(sqlText.trim())) {
 			throw new SqlParseException(DefaultFactory.getFactory().getMessages().getMissingSqlText() );
@@ -169,7 +173,7 @@ public Visitable visit(Visitable visitable) {
                     getSqlModel().addSelectListColumn(ref.getTableName(), ref.getColumnName());
             		break;
             	case NodeTypes.FROM_BASE_TABLE:
-            		if (!ynUpdateOrDelete) {//gotta skip this for updates/deletes, because foundationdb tells us about the table name twice for some unknown  (and probably very good) reasons.
+            		//if (!ynUpdateOrDelete) {//gotta skip this for updates/deletes, because Akiban tells us about the table name twice for some unknown  (and probably very good) reasons.
                 		FromBaseTable fromBaseTable = (FromBaseTable)node;
                 		try {
                 			TableName unkTableName = fromBaseTable.getOrigTableName();
@@ -180,7 +184,7 @@ public Visitable visit(Visitable visitable) {
                 		}
                 		myTable.setAlias(fromBaseTable.getCorrelationName());
                 		getSqlModel().addTable(myTable);
-            		}
+            		//}
             		break;
             	case NodeTypes.TABLE_NAME:
             		TableName unk2TableName = (TableName)node;

diff --git a/wuqiSpank/src/main/java/org/wuqispank/db/jsqlparser/JSqlParser.java b/wuqiSpank/src/main/java/org/wuqispank/db/jsqlparser/JSqlParser.java
@@ -13,6 +13,7 @@
 import org.wuqispank.db.SqlParseException;
 import org.wuqispank.model.ISqlModel;
 import org.wuqispank.model.ITable;
+import org.wuqispank.model.SqlType;
 
 public class JSqlParser implements ISqlParser {
 
@@ -24,19 +25,25 @@ public void parse(String sqlText) throws SqlParseException {
 		if (this.getSqlModel()==null) {
 			throw new SqlParseException(DefaultFactory.getFactory().getMessages().missingSqlModel(sqlText));
 		}
+		this.getSqlModel().setParser(this.getClass());
 
 		try {
 			Statement statement = CCJSqlParserUtil.parse(sqlText);
 			ParsedDataExtractor pde = null;
 			if (statement instanceof Select) {
 				pde = new SelectExtractor( (Select)statement, this.getSqlModel());
+				this.getSqlModel().setSqlType(SqlType.SELECT);
 			} else if (statement instanceof Update) {
 				pde = new UpdateExtractor( (Update)statement, this.getSqlModel());
+				this.getSqlModel().setSqlType(SqlType.UPDATE);
 			} else if (statement instanceof Insert) {
 				pde = new InsertExtractor( (Insert)statement, this.getSqlModel());
+				this.getSqlModel().setSqlType(SqlType.INSERT);
 			} else if (statement instanceof Delete) {
 				pde = new DeleteExtractor( (Delete)statement, this.getSqlModel());
+				this.getSqlModel().setSqlType(SqlType.DELETE);
 			} else {
+				this.getSqlModel().setSqlType(SqlType.OTHER);
 				throw new SqlParseException("The Replace statement is not yet supported.");
 			}
 			pde.extract();