Elasticsearch Client Plugin Contribution

agent-it/pom.xml

@@ -404,6 +404,12 @@
             <version>4.1.28.Final</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.bbossgroups.plugins</groupId>
+            <artifactId>bboss-elasticsearch-rest-jdbc</artifactId>
+            <version>5.6.9</version>
+            <scope>test</scope>
+        </dependency>
 
     </dependencies>
 

agent-it/src/test/java/com/navercorp/pinpoint/plugin/elasticsearchbboss/ElasticsearchExecutorIT.java

@@ -0,0 +1,52 @@
+/*
+ *  Copyright 2018 NAVER Corp.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.navercorp.pinpoint.plugin.elasticsearchbboss;
+
+
+import com.navercorp.pinpoint.bootstrap.plugin.test.PluginTestVerifier;
+import com.navercorp.pinpoint.bootstrap.plugin.test.PluginTestVerifierHolder;
+import com.navercorp.pinpoint.test.plugin.Dependency;
+import com.navercorp.pinpoint.test.plugin.PinpointPluginTestSuite;
+import org.frameworkset.elasticsearch.ElasticSearchHelper;
+import org.frameworkset.elasticsearch.client.ClientInterface;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+
+/**
+ * @author yinbp[yin-bp@163.com]
+ */
+@RunWith(PinpointPluginTestSuite.class)
+@Dependency({"com.bbossgroups.plugins:bboss-elasticsearch-rest-jdbc:5.6.9"})
+public class ElasticsearchExecutorIT {
+	@Test
+	public void test(){
+		try {
+			//build a elasticsearch client instance.
+			ClientInterface clientUtil = ElasticSearchHelper.getRestClientUtil();
+			//use the elasticsearch client instance to validate the indice exist or not
+			boolean existIndice = clientUtil.existIndice("twitter");
+//			BaseApplicationContext.shutdown();
+		}
+		catch (Exception e){
+
+		}
+
+		PluginTestVerifier verifier = PluginTestVerifierHolder.getInstance();
+		verifier.printCache();
+		verifier.verifyTraceCount(1);
+
+	}
+}

agent/src/main/resources-local/pinpoint.config

@@ -330,7 +330,10 @@ profiler.vertx.handler.base-packages=
 # HTTP server
 profiler.vertx.http.server.enable=true
 # Set HttpServerRequestHandler method name. The argument is io.vertx.core.http.HttpServerRequest.
-profiler.vertx.http.server.request-handler.method.name=io.vertx.ext.web.impl.RouterImpl.accept
+# Vert.x 3.6 ~ 3.7
+profiler.vertx.http.server.request-handler.method.name=io.vertx.ext.web.impl.RouterImpl.handle
+# Vert.x 3.5
+# profiler.vertx.http.server.request-handler.method.name=io.vertx.ext.web.impl.RouterImpl.accept
 profiler.vertx.http.server.tracerequestparam=true
 profiler.vertx.http.server.hidepinpointheader=true
 # URLs to exclude from tracing.
@@ -946,3 +949,14 @@ profiler.openwhisk.enable=true
 profiler.openwhisk.logging.message=true
 profiler.openwhisk.transform.targetname=org.apache.openwhisk.http.BasicHttpService.$anonfun$assignId$2
 profiler.openwhisk.transform.targetparameter=org.apache.openwhisk.http.BasicHttpService,boolean,akka.http.scaladsl.server.RequestContext
+
+###########################################################
+# ElasticsearchBBoss
+###########################################################
+profiler.elasticsearchbboss.enabled=true
+profiler.elasticsearchbboss.recordResult=false
+profiler.elasticsearchbboss.recordArgs=true
+profiler.elasticsearchbboss.recordDsl=true
+profiler.elasticsearchbboss.recordESVersion=true
+profiler.elasticsearchbboss.recordResponseHandlerClass=false
+profiler.elasticsearchbboss.maxDslSize=50000

agent/src/main/resources-release/pinpoint-real-env-lowoverhead-sample.config

@@ -740,3 +740,14 @@ profiler.hbase.client.enable=true
 profiler.hbase.client.admin.enable=false
 profiler.hbase.client.table.enable=false
 profiler.hbase.client.params.enable=false
+
+###########################################################
+# ElasticsearchBBoss
+###########################################################
+profiler.elasticsearchbboss.enabled=true
+profiler.elasticsearchbboss.recordResult=false
+profiler.elasticsearchbboss.recordArgs=true
+profiler.elasticsearchbboss.recordDsl=true
+profiler.elasticsearchbboss.recordESVersion=true
+profiler.elasticsearchbboss.recordResponseHandlerClass=false
+profiler.elasticsearchbboss.maxDslSize=50000

agent/src/main/resources-release/pinpoint.config

@@ -313,7 +313,10 @@ profiler.vertx.handler.base-packages=
 # HTTP server
 profiler.vertx.http.server.enable=true
 # Set HttpServerRequestHandler method name. The argument is io.vertx.core.http.HttpServerRequest.
-profiler.vertx.http.server.request-handler.method.name=io.vertx.ext.web.impl.RouterImpl.accept
+# Vert.x 3.6 ~ 3.7
+profiler.vertx.http.server.request-handler.method.name=io.vertx.ext.web.impl.RouterImpl.handle
+# Vert.x 3.5
+# profiler.vertx.http.server.request-handler.method.name=io.vertx.ext.web.impl.RouterImpl.accept
 profiler.vertx.http.server.tracerequestparam=true
 profiler.vertx.http.server.hidepinpointheader=true
 # URLs to exclude from tracing.
@@ -926,3 +929,14 @@ profiler.openwhisk.enable=false
 profiler.openwhisk.logging.message=false
 profiler.openwhisk.transform.targetname=org.apache.openwhisk.http.BasicHttpService.$anonfun$assignId$2
 profiler.openwhisk.transform.targetparameter=org.apache.openwhisk.http.BasicHttpService,boolean,akka.http.scaladsl.server.RequestContext
+
+###########################################################
+# ElasticsearchBBoss
+###########################################################
+profiler.elasticsearchbboss.enabled=true
+profiler.elasticsearchbboss.recordResult=false
+profiler.elasticsearchbboss.recordArgs=true
+profiler.elasticsearchbboss.recordDsl=true
+profiler.elasticsearchbboss.recordESVersion=true
+profiler.elasticsearchbboss.recordResponseHandlerClass=false
+profiler.elasticsearchbboss.maxDslSize=50000

collector/src/main/java/com/navercorp/pinpoint/collector/handler/grpc/GrpcSpanChunkHandler.java

@@ -52,10 +52,10 @@ private void handleSpanChunk(PSpanChunk pSpanChunk) {
         if (logger.isDebugEnabled()) {
             logger.debug("Handle PSpanChunk={}", pSpanChunk);
         }
-        AgentHeaderFactory.Header agentInfo = ServerContext.getAgentInfo();
-        SpanChunkBo spanChunkBo = spanFactory.buildSpanChunkBo(pSpanChunk, agentInfo);
 
         try {
+            final AgentHeaderFactory.Header agentInfo = ServerContext.getAgentInfo();
+            final SpanChunkBo spanChunkBo = spanFactory.buildSpanChunkBo(pSpanChunk, agentInfo);
             this.traceService.insertSpanChunk(spanChunkBo);
         } catch (Exception e) {
             logger.warn("SpanChunk handle error Caused:{}", e.getMessage(), e);

doc/modules.md

@@ -15,8 +15,8 @@
 |                                                                                   |                            |         |         |                        |
 | OpenWhisk                                                                         | whisk.core                 |         |         |                        | 
 |                                                                                   |                            |         |         |                        |
-| SpringMVC Framework                                                               | spring-webmvc              | 3.0.7   | 5.1.6   |                        | <AG>
-| Spring Web                                                                        | spring-web                 | 4.1.2   | 4.3.23  |                        | <AG>
+| SpringMVC Framework                                                               | spring-webmvc              | 3.0.7   | 5.1.7   |                        | <AG>
+| Spring Web                                                                        | spring-web                 | 4.1.2   | 4.3.24  |                        | <AG>
 | Spring RabbitMQ                                                                   | spring-rabbit              | 1.3.3   | 2.1.5   |                        | <AG>
 | Spring IBatis                                                                     | spring-ibatis              | 2.0.7   | 2.0.8   |                        | <AG>
 | Spring MyBatis                                                                    | mybatis-spring             | 1.1.0   | 1.3.2   |                        | <AG>

plugins/elasticsearch-bboss/README.md

@@ -0,0 +1,155 @@
+ElasticSearch is an Open Source (Apache 2), Distributed, RESTful, Search Engine built on top of Apache Lucene.
+
+This agent plugin is worked with elasticsearch bboss highlevel rest client [BBoss][bs].
+
+[BBoss][bs] is a best Java Highlevel Rest client for [ElasticSearch][es].
+
+
+If there is trace in the context, add context trace as span event.
+
+Some work pictures of this plugin:
+Elasticsearch Bboss work in tomcat
+![GitHub Logo](https://oscimg.oschina.net/oscnet/a6aa8b7e84db0437dd6cbff88bdf1160fab.jpg)
+![GitHub Logo](https://oscimg.oschina.net/oscnet/9665c0376579bbf1ca6093c1a0cf11c6c45.jpg)
+Async parallel slice scoll query
+![GitHub Logo](https://oscimg.oschina.net/oscnet/2ad63bcb0ad2de30a2cc13aa5f8a8ea86b4.jpg)
+Simple indice and indice type exist query
+![GitHub Logo](https://oscimg.oschina.net/oscnet/4c2e63e159786c28909ca2a003c8ee28432.jpg)
+dsl query
+![GitHub Logo](https://oscimg.oschina.net/oscnet/90fe224aee8b52c50b22fdfe0860658324d.jpg)
+
+
+How to use Elasticsearch BBoss.
+
+First add the maven dependency of BBoss to your pom.xml:
+
+```xml
+       <dependency>
+            <groupId>com.bbossgroups.plugins</groupId>
+            <artifactId>bboss-elasticsearch-rest-jdbc</artifactId>
+            <version>5.6.9</version>
+        </dependency>
+```
+
+If it's a spring boot project, you can replace the Maven coordinates above with the following Maven coordinates:
+
+```xml
+        <dependency>
+            <groupId>com.bbossgroups.plugins</groupId>
+            <artifactId>bboss-elasticsearch-spring-boot-starter</artifactId>
+            <version>5.6.9</version>
+        </dependency>
+```
+
+
+
+Next, add the Elasticsearch addresses to the application.properties file under the project resource directory, and create a new one if the file does not exist:
+
+```properties
+elasticsearch.rest.hostNames=10.21.20.168:9200
+
+#Cluster addresses are separated by commas
+
+#elasticsearch.rest.hostNames=10.180.211.27:9200,10.180.211.28:9200,10.180.211.29:9200
+```
+
+
+
+And last  create a jsp file named testElasticsearch.jsp :
+
+```jsp
+<%@ page import="org.frameworkset.elasticsearch.ElasticSearchHelper" %>
+<%@ page import="org.frameworkset.elasticsearch.client.ClientInterface" %>
+<%@ page import="org.frameworkset.elasticsearch.entity.ESDatas" %>
+<%@ page import="org.frameworkset.elasticsearch.scroll.ScrollHandler" %>
+<%@ page import="java.util.List" %>
+<%@ page import="java.util.Map" %>
+<%@ page import="org.frameworkset.elasticsearch.scroll.HandlerInfo" %>
+<%@ page language="java" pageEncoding="UTF-8"%>
+
+<%
+	ClientInterface clientUtil = ElasticSearchHelper.getRestClientUtil();
+	//get elasticsearch cluster state
+	String result = clientUtil.executeHttp("_cluster/state?pretty",ClientInterface.HTTP_GET);
+
+	//check indice twitter and index type tweet exist or not.
+	boolean exist1 = clientUtil.existIndiceType("twitter","tweet");
+	out.println("twitter  tweet type exist:"+exist1);
+	//check indice twitter exist or not
+	exist1 = clientUtil.existIndice("twitter");
+	out.println("twitter exist:"+exist1);
+	//count documents in indice twitter
+	long count = clientUtil.countAll("demo");
+	out.println(count);
+
+	//Get All documents of indice twitter,DEFAULT_FETCHSIZE is 5000
+	ESDatas<Map> esDatas = clientUtil.searchAll("demo", Map.class);
+
+	//Get All documents of indice twitter,Set fetchsize to 10000, Using ScrollHandler to process each batch of datas.
+	clientUtil.searchAll("demo",10000,new ScrollHandler<Map>() {
+		public void handle(ESDatas<Map> esDatas, HandlerInfo handlerInfo) throws Exception {
+			List<Map> dataList = esDatas.getDatas();
+			System.out.println("TotalSize:"+esDatas.getTotalSize());
+			if(dataList != null) {
+				System.out.println("dataList.size:" + dataList.size());
+			}
+			else
+			{
+				System.out.println("dataList.size:0");
+			}
+			//do something other such as do a db query.
+			//SQLExecutor.queryList(Map.class,"select * from td_sm_user");
+		}
+	},Map.class);
+    //Use slice parallel scoll query all documents of indice  twitter by 2 thread tasks. DEFAULT_FETCHSIZE is 5000
+	//You can also use ScrollHandler to process each batch of datas on your own.
+	clientUtil.searchAllParallel("demo", Map.class,2);
+	//use query dsl to execute scroll search and use ScrollHandler to parallel handle each page scroll search result.
+	clientUtil.scrollParallel("demo/_search","{ \"size\":10,\"query\": {\"match_all\": {}},\"sort\": [\"_doc\"]}","1m",Map.class,new ScrollHandler<Map>() {
+		public void handle(ESDatas<Map> esDatas, HandlerInfo handlerInfo) throws Exception {
+			List<Map> dataList = esDatas.getDatas();
+			System.out.println("TotalSize:"+esDatas.getTotalSize());
+			if(dataList != null) {
+				System.out.println("dataList.size:" + dataList.size());
+			}
+			else
+			{
+				System.out.println("dataList.size:0");
+			}
+			//do something other such as do a db query.
+			//SQLExecutor.queryList(Map.class,"select * from td_sm_user");
+		}
+	});
+	out.println("searchAllParallel:ok");
+%>
+
+```
+
+Put the file into the web project that has been connected to pinpoint, run the program, log on pinpoint to see the execution effect of bboss plugin.
+
+For more BBoss Elasticsearch Highlevel Rest Client detail see :
+ https://esdoc.bbossgroups.com/#/README
+
+github sourcecode:
+https://github.com/bbossgroups/bboss-elasticsearch
+
+Fast import bboss to your project:
+1. For Spring boot project see
+https://esdoc.bbossgroups.com/#/spring-booter-with-bboss
+2. For normal maven or gradle project see
+https://esdoc.bbossgroups.com/#/common-project-with-bboss
+
+ElasticsearchBBoss Demo
+
+[eshelloworld-booter][booter]
+
+[eshelloword-spring-boot-starter][springbooter]
+
+[booter]: https://github.com/bbossgroups/elasticsearch-example
+[springbooter]: https://github.com/bbossgroups/elasticsearch-springboot-example
+
+[bs]: https://github.com/bbossgroups/bboss-elasticsearch
+[es]: http://www.elasticsearch.org
+[DocumentCRUDTest]: https://github.com/bbossgroups/elasticsearch-example/blob/master/src/test/java/org/bboss/elasticsearchtest/crud/DocumentCRUDTest.java
+
+

plugins/elasticsearch-bboss/pom.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.navercorp.pinpoint</groupId>
+    <artifactId>pinpoint-plugins</artifactId>
+    <version>1.9.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>pinpoint-elasticsearch-bboss-plugin</artifactId>
+  <name>pinpoint-elasticsearch-bboss-plugin</name>
+  <packaging>jar</packaging>
+
+
+
+  <dependencies>
+
+    <dependency>
+      <groupId>com.navercorp.pinpoint</groupId>
+      <artifactId>pinpoint-bootstrap-core</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+  </dependencies>
+</project>

plugins/elasticsearch-bboss/src/main/java/com/navercorp/pinpoint/plugin/elasticsearchbboss/ElasticsearchConstants.java

@@ -0,0 +1,42 @@
+/*
+ *  Copyright 2018 NAVER Corp.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package com.navercorp.pinpoint.plugin.elasticsearchbboss;
+
+import com.navercorp.pinpoint.common.trace.*;
+
+import static com.navercorp.pinpoint.common.trace.ServiceTypeProperty.*;
+
+/**
+ * @author yinbp[yin-bp@163.com]
+ */
+public class ElasticsearchConstants {
+	public static final AnnotationKey ARGS_ANNOTATION_KEY = AnnotationKeyFactory.of(971, "es.args", AnnotationKeyProperty.VIEW_IN_RECORD_SET);
+	public static int maxDslSize = 50000;
+	public static final ServiceType ELASTICSEARCH = ServiceTypeFactory.of(9201, "ElasticsearchBBoss");
+
+	public static final AnnotationKey ARGS_URL_ANNOTATION_KEY = AnnotationKeyFactory.of(972, "es.url", AnnotationKeyProperty.VIEW_IN_RECORD_SET);
+	public static final AnnotationKey ARGS_DSL_ANNOTATION_KEY = AnnotationKeyFactory.of(973, "es.dsl", AnnotationKeyProperty.VIEW_IN_RECORD_SET);
+	public static final AnnotationKey ARGS_ACTION_ANNOTATION_KEY = AnnotationKeyFactory.of(974, "es.action", AnnotationKeyProperty.VIEW_IN_RECORD_SET);
+	public static final AnnotationKey ARGS_RESPONSEHANDLE_ANNOTATION_KEY = AnnotationKeyFactory.of(975, "es.responseHandle", AnnotationKeyProperty.VIEW_IN_RECORD_SET);
+	public static final AnnotationKey ARGS_VERSION_ANNOTATION_KEY = AnnotationKeyFactory.of(976, "es.version", AnnotationKeyProperty.VIEW_IN_RECORD_SET);
+	public static final ServiceType ELASTICSEARCH_EXECUTOR = ServiceTypeFactory.of(9202, "ElasticsearchBBossExecutor", AS_ALIAS);
+	public static final String ELASTICSEARCH_SCOPE = "ElasticsearchBBoss_SCOPE";
+	public static final String ELASTICSEARCH_Parallel_SCOPE = "ElasticsearchBBoss_Parallel_SCOPE";
+	public static final String ELASTICSEARCH_EXECUTOR_SCOPE = "ElasticsearchBBossExecutor_SCOPE";
+	public static final String[] clazzInterceptors = new String[]{
+			"org.frameworkset.elasticsearch.client.ConfigRestClientUtil",
+			"org.frameworkset.elasticsearch.client.RestClientUtil"
+	};
+}