JAV-416 expose and publish metrics using servo

common/common-rest/pom.xml

@@ -46,5 +46,9 @@
 			<artifactId>log4j</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>io.servicecomb</groupId>
+			<artifactId>foundation-metrics</artifactId>
+		</dependency>
 	</dependencies>
 </project>

common/common-rest/src/main/java/io/servicecomb/common/rest/RestProducerInvocation.java

@@ -34,6 +34,9 @@
 import io.servicecomb.core.definition.MicroserviceMeta;
 import io.servicecomb.core.definition.OperationMeta;
 import io.servicecomb.core.invocation.InvocationFactory;
+import io.servicecomb.foundation.metrics.MetricsServoRegistry;
+import io.servicecomb.foundation.metrics.performance.QueueMetrics;
+import io.servicecomb.foundation.metrics.performance.QueueMetricsData;
 import io.servicecomb.foundation.vertx.http.HttpServletRequestEx;
 import io.servicecomb.foundation.vertx.http.HttpServletResponseEx;
 import io.servicecomb.serviceregistry.RegistryUtils;
@@ -64,6 +67,7 @@ public void invoke(Transport transport, HttpServletRequestEx requestEx, HttpServ
 
   protected void scheduleInvocation() {
     OperationMeta operationMeta = restOperationMeta.getOperationMeta();
+    QueueMetrics metricsData = initMetrics(operationMeta);
     operationMeta.getExecutor().execute(() -> {
       synchronized (this.requestEx) {
         try {
@@ -76,7 +80,7 @@ protected void scheduleInvocation() {
             return;
           }
 
-          runOnExecutor();
+          runOnExecutor(metricsData);
         } catch (Throwable e) {
           LOGGER.error("rest server onRequest error", e);
           sendFailResponse(e);
@@ -85,11 +89,13 @@ protected void scheduleInvocation() {
     });
   }
 
-  protected void runOnExecutor() {
+  protected void runOnExecutor(QueueMetrics metricsData) {
     Object[] args = RestCodec.restToArgs(requestEx, restOperationMeta);
     this.invocation = InvocationFactory.forProvider(transport.getEndpoint(),
         restOperationMeta.getOperationMeta(),
         args);
+    this.invocation.setMetricsData(metricsData);
+    updateMetrics();
     invoke();
   }
 
@@ -118,6 +124,56 @@ protected RestOperationMeta findRestOperation() {
   protected void doInvoke() throws Throwable {
     invocation.next(resp -> {
       sendResponseQuietly(resp);
+      endMetrics();
+
     });
   }
+
+  /**
+   * Init the metrics. Note down the queue count and start time.
+   * @param operationMeta Operation data
+   * @return QueueMetrics
+   */
+  private QueueMetrics initMetrics(OperationMeta operationMeta) {
+    QueueMetrics metricsData = new QueueMetrics();
+    metricsData.setQueueStartTime(System.currentTimeMillis());
+    metricsData.setOperQualifiedName(operationMeta.getMicroserviceQualifiedName());
+    QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics()
+        .getOrCreateQueueMetrics(operationMeta.getMicroserviceQualifiedName());
+    reqQueue.incrementCountInQueue();
+    return metricsData;
+  }
+
+  /**
+   * Update the queue metrics.
+   */
+  private void updateMetrics() {
+    QueueMetrics metricsData = (QueueMetrics) this.invocation.getMetricsData();
+    if (null != metricsData) {
+      metricsData.setQueueEndTime(System.currentTimeMillis());
+      QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics()
+          .getOrCreateQueueMetrics(restOperationMeta.getOperationMeta().getMicroserviceQualifiedName());
+      reqQueue.incrementTotalCount();
+      Long timeInQueue = metricsData.getQueueEndTime() - metricsData.getQueueStartTime();
+      reqQueue.setTotalTime(reqQueue.getTotalTime() + timeInQueue);
+      reqQueue.setMinLifeTimeInQueue(timeInQueue);
+      reqQueue.setMaxLifeTimeInQueue(timeInQueue);
+      reqQueue.decrementCountInQueue();
+    }
+  }
+
+  /**
+   * Prepare the end time of queue metrics.
+   */
+  private void endMetrics() {
+    QueueMetrics metricsData = (QueueMetrics) this.invocation.getMetricsData();
+    if (null != metricsData) {
+      metricsData.setEndOperTime(System.currentTimeMillis());
+      QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics()
+          .getOrCreateQueueMetrics(restOperationMeta.getOperationMeta().getMicroserviceQualifiedName());
+      reqQueue.incrementTotalServExecutionCount();
+      reqQueue.setTotalServExecutionTime(
+          reqQueue.getTotalServExecutionTime() + (metricsData.getEndOperTime() - metricsData.getQueueEndTime()));
+    }
+  }
 }

common/common-rest/src/test/java/io/servicecomb/common/rest/TestRestProducerInvocation.java

@@ -48,6 +48,7 @@
 import io.servicecomb.core.definition.OperationMeta;
 import io.servicecomb.core.definition.SchemaMeta;
 import io.servicecomb.core.executor.ReactiveExecutor;
+import io.servicecomb.foundation.metrics.performance.QueueMetrics;
 import io.servicecomb.foundation.vertx.http.AbstractHttpServletRequest;
 import io.servicecomb.foundation.vertx.http.HttpServletRequestEx;
 import io.servicecomb.foundation.vertx.http.HttpServletResponseEx;
@@ -170,12 +171,14 @@ public void scheduleInvocationNormal(@Mocked OperationMeta operationMeta) {
         result = operationMeta;
         operationMeta.getExecutor();
         result = executor;
+        operationMeta.getMicroserviceQualifiedName();
+        result = "sayHi";
       }
     };
 
     restProducerInvocation = new MockUp<RestProducerInvocation>() {
       @Mock
-      void runOnExecutor() {
+      void runOnExecutor(QueueMetrics metricsData) {
         runOnExecutor = true;
       }
     }.getMockInstance();
@@ -199,6 +202,8 @@ public void scheduleInvocationTimeout(@Mocked OperationMeta operationMeta) {
         result = operationMeta;
         operationMeta.getExecutor();
         result = executor;
+        operationMeta.getMicroserviceQualifiedName();
+        result = "sayHi";
       }
     };
 
@@ -229,6 +234,8 @@ public void scheduleInvocationException(@Mocked OperationMeta operationMeta) {
         result = operationMeta;
         operationMeta.getExecutor();
         result = executor;
+        operationMeta.getMicroserviceQualifiedName();
+        result = "sayHi";
         requestEx.getAttribute(RestConst.REST_REQUEST);
         result = requestEx;
         RestCodec.restToArgs(requestEx, restOperationMeta);
@@ -255,6 +262,8 @@ public void runOnExecutor() {
     Object[] args = new Object[] {};
     new Expectations(RestCodec.class) {
       {
+        restOperationMeta.getOperationMeta().getMicroserviceQualifiedName();
+        result = "sayHi";
         RestCodec.restToArgs(requestEx, restOperationMeta);
         result = args;
       }
@@ -266,9 +275,7 @@ void invoke() {
       }
     }.getMockInstance();
     initRestProducerInvocation();
-
-    restProducerInvocation.runOnExecutor();
-
+    restProducerInvocation.runOnExecutor(new QueueMetrics());
     Assert.assertTrue(invokeNoParam);
     Assert.assertSame(args, restProducerInvocation.invocation.getSwaggerArguments());
   }
@@ -376,6 +383,8 @@ void handle(Invocation invocation, AsyncResponse asyncResp) throws Exception {
         result = schemaMeta;
         schemaMeta.getProviderHandlerChain();
         result = handlerChain;
+        operationMeta.getMicroserviceQualifiedName();
+        result = "sayHi";
       }
     };
 
@@ -388,8 +397,8 @@ protected void sendResponseQuietly(Response response) {
       }
     };
     initRestProducerInvocation();
+    invocation.setMetricsData(new QueueMetrics());
     restProducerInvocation.invocation = invocation;
-
     restProducerInvocation.doInvoke();
 
     Assert.assertSame(response, result.value);

core/src/main/java/io/servicecomb/core/Invocation.java

@@ -53,6 +53,17 @@ public class Invocation extends SwaggerInvocation {
   // 同步模式：避免应答在网络线程中处理解码等等业务级逻辑
   private Executor responseExecutor;
 
+  //start,end of queue and opertion time after queue for operation level metrics.
+  private Object metricsData;
+
+  public Object getMetricsData() {
+    return metricsData;
+  }
+
+  public void setMetricsData(Object metricsData) {
+    this.metricsData = metricsData;
+  }
+
   public Invocation(ReferenceConfig referenceConfig, OperationMeta operationMeta, Object[] swaggerArguments) {
     this.invocationType = InvocationType.CONSUMER;
     this.referenceConfig = referenceConfig;
@@ -75,7 +86,8 @@ private void init(OperationMeta operationMeta, Object[] swaggerArguments) {
 
   public Transport getTransport() {
     if (endpoint == null) {
-      throw new IllegalStateException("Endpoint is empty. Forget to configure \"loadbalance\" in consumer handler chain?");
+      throw new IllegalStateException(
+          "Endpoint is empty. Forget to configure \"loadbalance\" in consumer handler chain?");
     }
     return endpoint.getTransport();
   }

foundations/foundation-common/pom.xml

@@ -25,6 +25,14 @@
 	<artifactId>foundation-common</artifactId>
 
 	<dependencies>
+		<dependency>
+			<groupId>com.netflix.servo</groupId>
+			<artifactId>servo-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.netflix.hystrix</groupId>
+			<artifactId>hystrix-core</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>com.fasterxml.jackson.dataformat</groupId>
 			<artifactId>jackson-dataformat-xml</artifactId>

foundations/foundation-metrics/pom.xml

@@ -43,5 +43,13 @@
 			<artifactId>log4j</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>io.servicecomb</groupId>
+			<artifactId>swagger-invocation-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>io.servicecomb</groupId>
+			<artifactId>java-chassis-core</artifactId>
+		</dependency>
 	</dependencies>
 </project>