diff --git a/common/common-rest/pom.xml b/common/common-rest/pom.xml index ab3a439693d..4bab3535452 100644 --- a/common/common-rest/pom.xml +++ b/common/common-rest/pom.xml @@ -46,5 +46,9 @@ log4j test + + io.servicecomb + foundation-metrics + diff --git a/common/common-rest/src/main/java/io/servicecomb/common/rest/RestProducerInvocation.java b/common/common-rest/src/main/java/io/servicecomb/common/rest/RestProducerInvocation.java index 665154c4dfd..628fc5e8b79 100644 --- a/common/common-rest/src/main/java/io/servicecomb/common/rest/RestProducerInvocation.java +++ b/common/common-rest/src/main/java/io/servicecomb/common/rest/RestProducerInvocation.java @@ -34,6 +34,9 @@ import io.servicecomb.core.definition.MicroserviceMeta; import io.servicecomb.core.definition.OperationMeta; import io.servicecomb.core.invocation.InvocationFactory; +import io.servicecomb.foundation.metrics.MetricsServoRegistry; +import io.servicecomb.foundation.metrics.performance.QueueMetrics; +import io.servicecomb.foundation.metrics.performance.QueueMetricsData; import io.servicecomb.foundation.vertx.http.HttpServletRequestEx; import io.servicecomb.foundation.vertx.http.HttpServletResponseEx; import io.servicecomb.serviceregistry.RegistryUtils; @@ -64,6 +67,7 @@ public void invoke(Transport transport, HttpServletRequestEx requestEx, HttpServ protected void scheduleInvocation() { OperationMeta operationMeta = restOperationMeta.getOperationMeta(); + QueueMetrics metricsData = initMetrics(operationMeta); operationMeta.getExecutor().execute(() -> { synchronized (this.requestEx) { try { @@ -76,7 +80,7 @@ protected void scheduleInvocation() { return; } - runOnExecutor(); + runOnExecutor(metricsData); } catch (Throwable e) { LOGGER.error("rest server onRequest error", e); sendFailResponse(e); @@ -85,11 +89,13 @@ protected void scheduleInvocation() { }); } - protected void runOnExecutor() { + protected void runOnExecutor(QueueMetrics metricsData) { Object[] args = RestCodec.restToArgs(requestEx, restOperationMeta); this.invocation = InvocationFactory.forProvider(transport.getEndpoint(), restOperationMeta.getOperationMeta(), args); + this.invocation.setMetricsData(metricsData); + updateMetrics(); invoke(); } @@ -118,6 +124,56 @@ protected RestOperationMeta findRestOperation() { protected void doInvoke() throws Throwable { invocation.next(resp -> { sendResponseQuietly(resp); + endMetrics(); + }); } + + /** + * Init the metrics. Note down the queue count and start time. + * @param operationMeta Operation data + * @return QueueMetrics + */ + private QueueMetrics initMetrics(OperationMeta operationMeta) { + QueueMetrics metricsData = new QueueMetrics(); + metricsData.setQueueStartTime(System.currentTimeMillis()); + metricsData.setOperQualifiedName(operationMeta.getMicroserviceQualifiedName()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(operationMeta.getMicroserviceQualifiedName()); + reqQueue.incrementCountInQueue(); + return metricsData; + } + + /** + * Update the queue metrics. + */ + private void updateMetrics() { + QueueMetrics metricsData = (QueueMetrics) this.invocation.getMetricsData(); + if (null != metricsData) { + metricsData.setQueueEndTime(System.currentTimeMillis()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(restOperationMeta.getOperationMeta().getMicroserviceQualifiedName()); + reqQueue.incrementTotalCount(); + Long timeInQueue = metricsData.getQueueEndTime() - metricsData.getQueueStartTime(); + reqQueue.setTotalTime(reqQueue.getTotalTime() + timeInQueue); + reqQueue.setMinLifeTimeInQueue(timeInQueue); + reqQueue.setMaxLifeTimeInQueue(timeInQueue); + reqQueue.decrementCountInQueue(); + } + } + + /** + * Prepare the end time of queue metrics. + */ + private void endMetrics() { + QueueMetrics metricsData = (QueueMetrics) this.invocation.getMetricsData(); + if (null != metricsData) { + metricsData.setEndOperTime(System.currentTimeMillis()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(restOperationMeta.getOperationMeta().getMicroserviceQualifiedName()); + reqQueue.incrementTotalServExecutionCount(); + reqQueue.setTotalServExecutionTime( + reqQueue.getTotalServExecutionTime() + (metricsData.getEndOperTime() - metricsData.getQueueEndTime())); + } + } } diff --git a/common/common-rest/src/test/java/io/servicecomb/common/rest/TestRestProducerInvocation.java b/common/common-rest/src/test/java/io/servicecomb/common/rest/TestRestProducerInvocation.java index 5751a1e6dec..bd6f6517485 100644 --- a/common/common-rest/src/test/java/io/servicecomb/common/rest/TestRestProducerInvocation.java +++ b/common/common-rest/src/test/java/io/servicecomb/common/rest/TestRestProducerInvocation.java @@ -48,6 +48,7 @@ import io.servicecomb.core.definition.OperationMeta; import io.servicecomb.core.definition.SchemaMeta; import io.servicecomb.core.executor.ReactiveExecutor; +import io.servicecomb.foundation.metrics.performance.QueueMetrics; import io.servicecomb.foundation.vertx.http.AbstractHttpServletRequest; import io.servicecomb.foundation.vertx.http.HttpServletRequestEx; import io.servicecomb.foundation.vertx.http.HttpServletResponseEx; @@ -170,12 +171,14 @@ public void scheduleInvocationNormal(@Mocked OperationMeta operationMeta) { result = operationMeta; operationMeta.getExecutor(); result = executor; + operationMeta.getMicroserviceQualifiedName(); + result = "sayHi"; } }; restProducerInvocation = new MockUp() { @Mock - void runOnExecutor() { + void runOnExecutor(QueueMetrics metricsData) { runOnExecutor = true; } }.getMockInstance(); @@ -199,6 +202,8 @@ public void scheduleInvocationTimeout(@Mocked OperationMeta operationMeta) { result = operationMeta; operationMeta.getExecutor(); result = executor; + operationMeta.getMicroserviceQualifiedName(); + result = "sayHi"; } }; @@ -229,6 +234,8 @@ public void scheduleInvocationException(@Mocked OperationMeta operationMeta) { result = operationMeta; operationMeta.getExecutor(); result = executor; + operationMeta.getMicroserviceQualifiedName(); + result = "sayHi"; requestEx.getAttribute(RestConst.REST_REQUEST); result = requestEx; RestCodec.restToArgs(requestEx, restOperationMeta); @@ -255,6 +262,8 @@ public void runOnExecutor() { Object[] args = new Object[] {}; new Expectations(RestCodec.class) { { + restOperationMeta.getOperationMeta().getMicroserviceQualifiedName(); + result = "sayHi"; RestCodec.restToArgs(requestEx, restOperationMeta); result = args; } @@ -266,9 +275,7 @@ void invoke() { } }.getMockInstance(); initRestProducerInvocation(); - - restProducerInvocation.runOnExecutor(); - + restProducerInvocation.runOnExecutor(new QueueMetrics()); Assert.assertTrue(invokeNoParam); Assert.assertSame(args, restProducerInvocation.invocation.getSwaggerArguments()); } @@ -376,6 +383,8 @@ void handle(Invocation invocation, AsyncResponse asyncResp) throws Exception { result = schemaMeta; schemaMeta.getProviderHandlerChain(); result = handlerChain; + operationMeta.getMicroserviceQualifiedName(); + result = "sayHi"; } }; @@ -388,8 +397,8 @@ protected void sendResponseQuietly(Response response) { } }; initRestProducerInvocation(); + invocation.setMetricsData(new QueueMetrics()); restProducerInvocation.invocation = invocation; - restProducerInvocation.doInvoke(); Assert.assertSame(response, result.value); diff --git a/core/src/main/java/io/servicecomb/core/Invocation.java b/core/src/main/java/io/servicecomb/core/Invocation.java index 0e78025d1a1..fff523aa816 100644 --- a/core/src/main/java/io/servicecomb/core/Invocation.java +++ b/core/src/main/java/io/servicecomb/core/Invocation.java @@ -53,6 +53,17 @@ public class Invocation extends SwaggerInvocation { // 同步模式：避免应答在网络线程中处理解码等等业务级逻辑 private Executor responseExecutor; + //start,end of queue and opertion time after queue for operation level metrics. + private Object metricsData; + + public Object getMetricsData() { + return metricsData; + } + + public void setMetricsData(Object metricsData) { + this.metricsData = metricsData; + } + public Invocation(ReferenceConfig referenceConfig, OperationMeta operationMeta, Object[] swaggerArguments) { this.invocationType = InvocationType.CONSUMER; this.referenceConfig = referenceConfig; @@ -75,7 +86,8 @@ private void init(OperationMeta operationMeta, Object[] swaggerArguments) { public Transport getTransport() { if (endpoint == null) { - throw new IllegalStateException("Endpoint is empty. Forget to configure \"loadbalance\" in consumer handler chain?"); + throw new IllegalStateException( + "Endpoint is empty. Forget to configure \"loadbalance\" in consumer handler chain?"); } return endpoint.getTransport(); } diff --git a/foundations/foundation-common/pom.xml b/foundations/foundation-common/pom.xml index 66f3e9fd450..add23b6ef3f 100644 --- a/foundations/foundation-common/pom.xml +++ b/foundations/foundation-common/pom.xml @@ -25,6 +25,14 @@ foundation-common + + com.netflix.servo + servo-core + + + com.netflix.hystrix + hystrix-core + com.fasterxml.jackson.dataformat jackson-dataformat-xml diff --git a/foundations/foundation-metrics/pom.xml b/foundations/foundation-metrics/pom.xml index 3a10fb74f04..30ddb04ded4 100644 --- a/foundations/foundation-metrics/pom.xml +++ b/foundations/foundation-metrics/pom.xml @@ -43,5 +43,13 @@ log4j test + + io.servicecomb + swagger-invocation-core + + + io.servicecomb + java-chassis-core + diff --git a/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/MetricsServoRegistry.java b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/MetricsServoRegistry.java new file mode 100644 index 00000000000..718becf682a --- /dev/null +++ b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/MetricsServoRegistry.java @@ -0,0 +1,593 @@ +/* + * Copyright 2017 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.servicecomb.foundation.metrics; + +import java.io.File; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.lang.management.OperatingSystemMXBean; +import java.lang.management.ThreadMXBean; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Vector; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.InitializingBean; + +import com.netflix.config.DynamicPropertyFactory; +import com.netflix.hystrix.HystrixCommandMetrics; +import com.netflix.hystrix.HystrixEventType; +import com.netflix.servo.DefaultMonitorRegistry; +import com.netflix.servo.annotations.DataSourceType; +import com.netflix.servo.monitor.AbstractMonitor; +import com.netflix.servo.monitor.BasicCompositeMonitor; +import com.netflix.servo.monitor.Gauge; +import com.netflix.servo.monitor.Informational; +import com.netflix.servo.monitor.Monitor; +import com.netflix.servo.monitor.MonitorConfig; +import com.netflix.servo.publish.BasicMetricFilter; +import com.netflix.servo.publish.CounterToRateMetricTransform; +import com.netflix.servo.publish.FileMetricObserver; +import com.netflix.servo.publish.MetricObserver; +import com.netflix.servo.publish.MonitorRegistryMetricPoller; +import com.netflix.servo.publish.PollRunnable; +import com.netflix.servo.publish.PollScheduler; + +import io.servicecomb.foundation.metrics.performance.MetricsDataMonitor; +import io.servicecomb.foundation.metrics.performance.QueueMetricsData; +import rx.functions.Func0; + +/** + * Implementation of metrics preparation and servo registry. + */ +public class MetricsServoRegistry implements InitializingBean { + + protected static final ThreadLocal LOCAL_METRICS_MAP = new ThreadLocal<>(); + + private static final String METRICS_POLL_TIME = "cse.metrics.polltime"; + + private static final String FILENAME = "cse.metrics.file.name"; + + private static final String FILEPATH = "cse.metrics.file.path"; + + private MetricsDataMonitor localMetrics = new MetricsDataMonitor(); + + private static Vector metricsList = new Vector<>(); + + /* + * Added getter for unit test of local metrics. + * + * @return Local metric reference + */ + public MetricsDataMonitor getLocalMetrics() { + return localMetrics; + } + + /** + * Get or create local metrics. + * @return MetricsDataMonitor + */ + public static MetricsDataMonitor getOrCreateLocalMetrics() { + MetricsDataMonitor metricsDataMonitor = LOCAL_METRICS_MAP.get(); + if (metricsDataMonitor == null) { + metricsDataMonitor = new MetricsDataMonitor(); + LOCAL_METRICS_MAP.set(metricsDataMonitor); + metricsList.add(metricsDataMonitor); + } + return metricsDataMonitor; + } + + /** + * Get the initial metrics and register with servo. + */ + public void initMetricsPublishing() { + + /* list of monitors */ + List> monitors = getMetricsMonitors(); + MonitorConfig commandMetricsConfig = MonitorConfig.builder("metrics").build(); + BasicCompositeMonitor commandMetricsMonitor = new BasicCompositeMonitor(commandMetricsConfig, monitors); + DefaultMonitorRegistry.getInstance().register(commandMetricsMonitor); + PollScheduler scheduler = PollScheduler.getInstance(); + if (!scheduler.isStarted()) { + scheduler.start(); + } + + int metricPoll = DynamicPropertyFactory.getInstance().getIntProperty(METRICS_POLL_TIME, 60).get(); + String fileName = DynamicPropertyFactory.getInstance().getStringProperty(FILENAME, "metrics").get(); + String filePath = DynamicPropertyFactory.getInstance().getStringProperty(FILEPATH, ".").get(); + MetricObserver fileObserver = new FileMetricObserver(fileName, new File(filePath)); + MetricObserver transform = new CounterToRateMetricTransform(fileObserver, metricPoll, TimeUnit.SECONDS); + PollRunnable task = new PollRunnable(new MonitorRegistryMetricPoller(), BasicMetricFilter.MATCH_ALL, transform); + scheduler.addPoller(task, metricPoll, TimeUnit.SECONDS); + } + + @Override + public void afterPropertiesSet() throws Exception { + initMetricsPublishing(); + } + + /** + * Get instance level total requests by comparing the last saved data. + */ + protected final Func0 getTotalReqProvider = new Func0() { + @Override + public Number call() { + Long totalReqProvider = 0L; + for (MetricsDataMonitor metricsDataMonitor : metricsList) { + totalReqProvider += metricsDataMonitor.getTotalReqProvider(); + } + Long metricValue = totalReqProvider - localMetrics.getTotalReqProvider(); + localMetrics.setTotalReqProvider(totalReqProvider); + return metricValue; + } + }; + + /** + * Get instance level total failed requests by comparing the last saved data. + */ + protected final Func0 getTotalFailedReqProvider = new Func0() { + @Override + public Number call() { + Long totalFailedReqProvider = 0L; + for (MetricsDataMonitor metricsDataMonitor : metricsList) { + totalFailedReqProvider += metricsDataMonitor.getTotalFailReqProvider(); + } + Long metricValue = totalFailedReqProvider - localMetrics.getTotalFailReqProvider(); + localMetrics.setTotalFailReqProvider(totalFailedReqProvider); + return metricValue; + } + }; + + /** + * Get instance level total failed requests by comparing the last saved data. + */ + protected final Func0 getTotalReqConsumer = new Func0() { + @Override + public Number call() { + Long totalReqConsumer = 0L; + for (MetricsDataMonitor metricsDataMonitor : metricsList) { + totalReqConsumer += metricsDataMonitor.getTotalReqConsumer(); + } + Long metricValue = totalReqConsumer - localMetrics.getTotalReqConsumer(); + localMetrics.setTotalReqConsumer(totalReqConsumer); + return metricValue; + } + }; + + /** + * Get instance level total failed requests by comparing the last saved data. + */ + protected final Func0 getFailedTotalReqConsumer = new Func0() { + @Override + public Number call() { + Long totalFailedReqConsumer = 0L; + for (MetricsDataMonitor metricsDataMonitor : metricsList) { + totalFailedReqConsumer += metricsDataMonitor.getTotalFailReqConsumer(); + } + Long metricValue = totalFailedReqConsumer - localMetrics.getTotalFailReqConsumer(); + localMetrics.setTotalFailReqConsumer(totalFailedReqConsumer); + return metricValue; + } + }; + + /** + * Get operational level total request and total failed requests by comparing the + * last saved data. + */ + protected final Func0 getTotalReqProdOperLevel = new Func0() { + @Override + public String call() { + Map totalMap = new HashMap(); + Map oldMap = localMetrics.operMetricsTotalReq; + Map metricMap = new HashMap(); + for (MetricsDataMonitor metricsDataMonitor : metricsList) { + Collection keySet = metricsDataMonitor.operMetricsTotalReq.keySet(); + for (String key : keySet) { + Long value = totalMap.get(key); + if (null == value) { + totalMap.put(key, metricsDataMonitor.getOperMetTotalReq(key)); + } else { + totalMap.put(key, metricsDataMonitor.getOperMetTotalReq(key) + value); + } + } + } + Collection keySet = totalMap.keySet(); + for (String key : keySet) { + if (oldMap.containsKey(key)) { + metricMap.put(key, totalMap.get(key) - oldMap.get(key)); + } else { + metricMap.put(key, totalMap.get(key)); + } + } + localMetrics.operMetricsTotalReq.putAll(totalMap); + return metricMap.toString(); + } + }; + + /** + * Get operational level total request and total failed requets by comparing the + * last saved data. + */ + protected final Func0 getTotalReqFailProdOperLevel = new Func0() { + @Override + public String call() { + Map totalMap = new HashMap(); + Map oldMap = localMetrics.operMetricsTotalFailReq; + Map metricMap = new HashMap(); + for (MetricsDataMonitor metricsDataMonitor : metricsList) { + Collection keySet = metricsDataMonitor.operMetricsTotalFailReq.keySet(); + for (String key : keySet) { + Long value = totalMap.get(key); + if (null == value) { + totalMap.put(key, metricsDataMonitor.getOperMetTotalFailReq(key)); + } else { + totalMap.put(key, metricsDataMonitor.getOperMetTotalFailReq(key) + value); + } + } + } + Collection keySet = totalMap.keySet(); + for (String key : keySet) { + if (oldMap.containsKey(key)) { + metricMap.put(key, totalMap.get(key) - oldMap.get(key)); + } else { + metricMap.put(key, totalMap.get(key)); + } + } + localMetrics.operMetricsTotalFailReq.putAll(totalMap); + return metricMap.toString(); + } + }; + + /** + * Get operational level/instance level queue related metrics by comparing the + * last saved data. + */ + protected final Func0 getQueueMetrics = new Func0() { + @Override + public String call() { + Map totalMap = new HashMap(); + + for (MetricsDataMonitor metricsDataMonitor : metricsList) { + Collection keySet = metricsDataMonitor.getQueueMetrics().keySet(); + for (String key : keySet) { + QueueMetricsData value = totalMap.get(key); + if (null == value) { + totalMap.put(key, metricsDataMonitor.getQueueMetrics().get(key)); + } else { + QueueMetricsData newValue = metricsDataMonitor.getQueueMetrics().get(key); + QueueMetricsData totalValue = new QueueMetricsData(); + totalValue.setCountInQueue(newValue.getCountInQueue() + value.getCountInQueue()); + totalValue.setTotalTime(newValue.getTotalTime() + value.getTotalTime()); + totalValue.setTotalCount(newValue.getTotalCount() + value.getTotalCount()); + totalValue + .setTotalServExecutionTime(newValue.getTotalServExecutionTime() + value.getTotalServExecutionTime()); + totalValue + .setTotalServExecutionCount(newValue.getTotalServExecutionCount() + value.getTotalServExecutionCount()); + if ((value.getMinLifeTimeInQueue() <= 0) + || (newValue.getMinLifeTimeInQueue() < value.getMinLifeTimeInQueue())) { + totalValue.setMinLifeTimeInQueue(newValue.getMinLifeTimeInQueue()); + } + newValue.resetMinLifeTimeInQueue(); + if ((value.getMaxLifeTimeInQueue() <= 0) + || (newValue.getMaxLifeTimeInQueue() > value.getMaxLifeTimeInQueue())) { + totalValue.setMaxLifeTimeInQueue(newValue.getMaxLifeTimeInQueue()); + } + newValue.resetMaxLifeTimeInQueue(); + totalMap.put(key, totalValue); + } + } + } + + Map oldMap = localMetrics.getQueueMetrics(); + Map metricMap = new HashMap(); + Map result = new HashMap<>(); + Map resultInstancePublishMap = new HashMap<>(); + + QueueMetricsData totalValueInstance = new QueueMetricsData(); + + Collection keySet = totalMap.keySet(); + Map resultMap; + + for (String key : keySet) { + resultMap = new HashMap<>(); + if (oldMap.containsKey(key)) { + QueueMetricsData newValue = new QueueMetricsData(); + QueueMetricsData totalValue = totalMap.get(key); + QueueMetricsData oldValue = oldMap.get(key); + newValue.setCountInQueue(totalValue.getCountInQueue()); + newValue.setTotalTime(totalValue.getTotalTime() - oldValue.getTotalTime()); + newValue.setTotalCount(totalValue.getTotalCount() - oldValue.getTotalCount()); + newValue + .setTotalServExecutionTime(totalValue.getTotalServExecutionTime() - oldValue.getTotalServExecutionTime()); + newValue.setTotalServExecutionCount( + totalValue.getTotalServExecutionCount() - oldValue.getTotalServExecutionCount()); + newValue.setMinLifeTimeInQueue(totalValue.getMinLifeTimeInQueue()); + newValue.setMaxLifeTimeInQueue(totalValue.getMaxLifeTimeInQueue()); + metricMap.put(key, newValue); + } else { + metricMap.put(key, totalMap.get(key)); + } + + resultMap.put("countInQueue", metricMap.get(key).getCountInQueue().toString()); + + long count = metricMap.get(key).getTotalCount(); + double avgTimeInQueue = 0; + if (count > 0) { + avgTimeInQueue = metricMap.get(key).getTotalTime() / count; + } + resultMap.put("AverageTimeInQueue", String.valueOf(avgTimeInQueue)); + long countService = metricMap.get(key).getTotalServExecutionCount(); + double avgServiceTimeInQueue = 0; + if (countService > 0) { + avgServiceTimeInQueue = metricMap.get(key).getTotalServExecutionTime() / countService; + } + resultMap.put("AverageServiceExecutionTime", String.valueOf(avgServiceTimeInQueue)); + resultMap.put("MinLifeTimeInQueue", metricMap.get(key).getMinLifeTimeInQueue().toString()); + resultMap.put("MaxLifeTimeInQueue", metricMap.get(key).getMaxLifeTimeInQueue().toString()); + + result.put(key, resultMap.toString()); + + //get the all values for instance level. + totalValueInstance.setCountInQueue(metricMap.get(key).getCountInQueue()); + totalValueInstance.setTotalTime(totalValueInstance.getTotalTime() + metricMap.get(key).getTotalTime()); + totalValueInstance.setTotalCount(totalValueInstance.getTotalCount() + metricMap.get(key).getTotalCount()); + totalValueInstance + .setTotalServExecutionTime( + totalValueInstance.getTotalServExecutionTime() + metricMap.get(key).getTotalServExecutionTime()); + totalValueInstance + .setTotalServExecutionCount( + totalValueInstance.getTotalServExecutionCount() + metricMap.get(key).getTotalServExecutionCount()); + + if (totalValueInstance.getMinLifeTimeInQueue() <= 0 + || metricMap.get(key).getMinLifeTimeInQueue() < totalValueInstance.getMinLifeTimeInQueue()) { + totalValueInstance.setMinLifeTimeInQueue(metricMap.get(key).getMinLifeTimeInQueue()); + } + if (totalValueInstance.getMaxLifeTimeInQueue() <= 0 + || totalMap.get(key).getMaxLifeTimeInQueue() > totalValueInstance.getMaxLifeTimeInQueue()) { + totalValueInstance.setMaxLifeTimeInQueue(metricMap.get(key).getMaxLifeTimeInQueue()); + } + + localMetrics.setQueueMetrics(new ConcurrentHashMap<>(totalMap)); + } + + //prepare the result map for instance level. + resultInstancePublishMap.put("countInQueue", totalValueInstance.getCountInQueue().toString()); + long countInst = totalValueInstance.getTotalCount(); + double avgTimeInQueueIns = 0; + if (countInst > 0) { + avgTimeInQueueIns = totalValueInstance.getTotalTime() / countInst; + } + resultInstancePublishMap.put("AverageTimeInQueue", String.valueOf(avgTimeInQueueIns)); + long countServiceInst = totalValueInstance.getTotalServExecutionCount(); + double avgServiceTimeInQueueInst = 0; + if (countServiceInst > 0) { + avgServiceTimeInQueueInst = totalValueInstance.getTotalServExecutionTime() / countServiceInst; + } + resultInstancePublishMap.put("AverageServiceExecutionTime", String.valueOf(avgServiceTimeInQueueInst)); + resultInstancePublishMap.put("MinLifeTimeInQueue", totalValueInstance.getMinLifeTimeInQueue().toString()); + resultInstancePublishMap.put("MaxLifeTimeInQueue", totalValueInstance.getMaxLifeTimeInQueue().toString()); + result.put("InstanceLevel", resultInstancePublishMap.toString()); + + return result.toString(); + } + }; + + /** + * Get CPU and memory information metrics. + */ + protected final Func0 getCpuAndMemory = new Func0() { + @Override + public String call() { + Map memoryMap = new HashMap<>(); + OperatingSystemMXBean osMxBean = ManagementFactory.getOperatingSystemMXBean(); + double cpu = osMxBean.getSystemLoadAverage(); + memoryMap.put("CPU System Load", String.valueOf(cpu)); + + ThreadMXBean threadmxBean = ManagementFactory.getThreadMXBean(); + int threadCount = threadmxBean.getThreadCount(); + memoryMap.put("CPU Current Running Threads", String.valueOf(threadCount)); + + MemoryMXBean memBean = ManagementFactory.getMemoryMXBean(); + MemoryUsage memHeapUsage = memBean.getHeapMemoryUsage(); + MemoryUsage nonHeapUsage = memBean.getNonHeapMemoryUsage(); + memoryMap.put("heapInit", String.valueOf(memHeapUsage.getInit())); + memoryMap.put("heapMax", String.valueOf(memHeapUsage.getMax())); + memoryMap.put("heapCommit", String.valueOf(memHeapUsage.getCommitted())); + memoryMap.put("heapUsed", String.valueOf(memHeapUsage.getUsed())); + memoryMap.put("nonHeapInit", String.valueOf(nonHeapUsage.getInit())); + memoryMap.put("nonHeapMax", String.valueOf(nonHeapUsage.getMax())); + memoryMap.put("nonHeapCommit", String.valueOf(nonHeapUsage.getCommitted())); + memoryMap.put("nonHeapUsed", String.valueOf(nonHeapUsage.getUsed())); + return memoryMap.toString(); + } + }; + + /** + * Get TPS and latency for operational and instance level from hystrix. + */ + protected final Func0 getTpsAndLatency = new Func0() { + @Override + public String call() { + Map tpsAndLatencyMap = new HashMap<>(); + Collection instances = HystrixCommandMetrics.getInstances(); + + long insTotalTps = 0; + long insTotalLatency = 0; + long cumulativeTotalCount = 0; + + for (HystrixCommandMetrics instance : instances) { + long successCount = instance.getRollingCount(HystrixEventType.SUCCESS); + long failureCount = instance.getRollingCount(HystrixEventType.FAILURE); + int operLatency = instance.getExecutionTimeMean(); + long totalCallCount = successCount + failureCount; + cumulativeTotalCount += totalCallCount; + int windowTime = instance.getProperties().metricsRollingStatisticalWindowInMilliseconds().get() / 1000; + double qpsVal = (double) (totalCallCount) / windowTime; + BigDecimal bigDecimal = new BigDecimal(qpsVal); + BigDecimal bigDecimalVal = bigDecimal.setScale(1, RoundingMode.HALF_DOWN); + Double tpsOper = bigDecimalVal.doubleValue(); + tpsAndLatencyMap.put("TPS-" + instance.getCommandKey().name(), String.valueOf(tpsOper)); + tpsAndLatencyMap.put("Latency-" + instance.getCommandKey().name(), String.valueOf(operLatency)); + insTotalTps += tpsOper; + insTotalLatency += operLatency; + } + + double instanceLatency = (double) (insTotalLatency) / cumulativeTotalCount; + + tpsAndLatencyMap.put("TPS Instance_Level", String.valueOf(insTotalTps)); + tpsAndLatencyMap.put("Latency Instance_Level", String.valueOf(instanceLatency)); + + return tpsAndLatencyMap.toString(); + } + }; + + /** + * Implementation of request metrics with using servo guage metric type. + */ + protected abstract class GaugeMetric extends AbstractMonitor implements Gauge { + + public GaugeMetric(MonitorConfig config) { + super(config.withAdditionalTag(DataSourceType.GAUGE)); + } + + @Override + public Number getValue(int n) { + return getValue(); + } + + @Override + public abstract Number getValue(); + } + + /** + * Implementation of queue average metrics with using servo information metric + * type. + */ + protected abstract class InformationalMetric extends AbstractMonitor implements Informational { + public InformationalMetric(MonitorConfig config) { + super(config.withAdditionalTag(DataSourceType.INFORMATIONAL)); + } + + @Override + public String getValue(int n) { + return getValue(); + } + + @Override + public abstract String getValue(); + } + + /** + * Get the total requests and failed requests for instance level. + * + * @param metricsName Name of the metrics + * @param instance object of latest metrics + * @param fieldName metric field + * @param metricToEvaluate observable method to be called for preparation of metrics. + * @return Guage metrics + */ + protected Monitor getRequestValuesGaugeMonitor(final String metricsName, + final Func0 metricToEvaluate) { + return new GaugeMetric(MonitorConfig.builder(metricsName).build()) { + + @Override + public Number getValue() { + return metricToEvaluate.call(); + } + }; + } + + /** + * Get the total requests and failed requests for each producer. + * + * @param metricsName Name of the metrics + * @param instance object of latest metrics + * @param fieldName metric field + * @param metricToEvaluate observable method to be called for preparation of metrics. + * @return Guage metrics + */ + protected Monitor getInfoMetricsOperationLevel(final String metricsName, + final Func0 metricToEvaluate) { + return new InformationalMetric(MonitorConfig.builder(metricsName).build()) { + @Override + public String getValue() { + return metricToEvaluate.call(); + } + }; + } + + /** + * Get the total requests and failed requests for each producer. + * + * @param metricsName Name of the metrics + * @param instance object of latest metrics + * @param fieldName metric field + * @param metricToEvaluate observable method to be called for preparation of metrics. + * @return Guage metrics + */ + protected Monitor getInfoMetricsOperationalAndInstance(final String name, + final Func0 metricToEvaluate) { + return new InformationalMetric(MonitorConfig.builder(name).build()) { + @Override + public String getValue() { + return metricToEvaluate.call(); + } + }; + } + + /** + * Prepare the initial metrics. + * + * @return List of monitors + */ + private List> getMetricsMonitors() { + + List> monitors = new ArrayList>(); + monitors.add(getRequestValuesGaugeMonitor("TotalRequestsPerProvider INSTANCE_LEVEL", + getTotalReqProvider)); + + monitors.add(getRequestValuesGaugeMonitor("TotalFailedRequestsPerProvider INSTANCE_LEVEL", + getTotalFailedReqProvider)); + + monitors.add(getRequestValuesGaugeMonitor("TotalRequestsPerConsumer INSTANCE_LEVEL", + getTotalReqConsumer)); + + monitors.add(getRequestValuesGaugeMonitor("TotalFailRequestsPerConsumer INSTANCE_LEVEL", + getFailedTotalReqConsumer)); + + monitors.add(getInfoMetricsOperationLevel("TotalRequestProvider OPERATIONAL_LEVEL", + getTotalReqProdOperLevel)); + + monitors.add(getInfoMetricsOperationLevel("TotalFailedRequestProvider OPERATIONAL_LEVEL", + getTotalReqFailProdOperLevel)); + + monitors.add(getInfoMetricsOperationalAndInstance("RequestQueueRelated", getQueueMetrics)); + + monitors.add(getInfoMetricsOperationalAndInstance("TPS and Latency", getTpsAndLatency)); + + monitors.add(getInfoMetricsOperationalAndInstance("CPU and Memory", getCpuAndMemory)); + + return monitors; + } +} diff --git a/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/MetricsDataMonitor.java b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/MetricsDataMonitor.java new file mode 100644 index 00000000000..afe7054cc90 --- /dev/null +++ b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/MetricsDataMonitor.java @@ -0,0 +1,236 @@ +/* + * Copyright 2017 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.servicecomb.foundation.metrics.performance; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Implements the collection of metrics such as total request, total fail + * request and average times for requests. + */ +public class MetricsDataMonitor { + + // inc + // invocation start + // succ + fail + public Long totalReqProvider = new Long(0); + + // inc + // after invocation finished + public Long totalFailReqProvider = new Long(0); + + // inc + // after invocation start + public Long totalReqConsumer = new Long(0); + + // inc + // after invocation finished + public Long totalFailReqConsumer = new Long(0); + + // key is operQualifiedName + // inc + // after invocation finished + public Map operMetricsTotalReq = new ConcurrentHashMap(); + + // key is operQualifiedName + // inc + // after invocation finished + public Map operMetricsTotalFailReq = new ConcurrentHashMap(); + + // key is operQualifiedName + public Map queueMetrics = new ConcurrentHashMap(); + + /** + * default constructor. + */ + public MetricsDataMonitor() { + + } + + /** + * Returns the map of average values for both instance and operational level. + * + * @param pathId Operation path id + * @return QueueMetrics object based on key + */ + public QueueMetricsData getOrCreateQueueMetrics(String pathId) { + return queueMetrics.computeIfAbsent(pathId, p -> { + return new QueueMetricsData(); + }); + } + + /** + * Returns the map of average values for both instance and operational level. + * + * @return queue metrics map + */ + public Map getQueueMetrics() { + return queueMetrics; + } + + /** + * Returns the map of average values for both instance and operational level. + * + * @return queue metrics map + */ + public Map setQueueMetrics(Map newMap) { + return queueMetrics = newMap; + } + + /** + * Sets the map of average values for both instance and operational levels. + * + * @param pathId Operation path id + * @param reqQueue RequestQueue + */ + public void setQueueMetrics(String pathId, QueueMetricsData reqQueue) { + this.queueMetrics.put(pathId, reqQueue); + } + + /** + * Returns the total requests per provider. + * + * @return long total requests for provider + */ + public long getTotalReqProvider() { + return totalReqProvider; + } + + /** + * Increments the total requests per provider. + */ + public void incrementTotalReqProvider() { + this.totalReqProvider++; + } + + /** + * Sets the total requests per provider. + */ + public void setTotalReqProvider(Long totalReqProvider) { + this.totalReqProvider = totalReqProvider; + } + + /** + * Returns the total fail requests per provider. + * + * @return long total failed requests for provider + */ + public long getTotalFailReqProvider() { + return totalFailReqProvider; + } + + /** + * Sets the total fail requests per provider. + */ + public void incrementTotalFailReqProvider() { + this.totalFailReqProvider++; + } + + /** + * Sets the total failed requests per provider. + */ + public void setTotalFailReqProvider(Long totalFailedReqProvider) { + this.totalFailReqProvider = totalFailedReqProvider; + } + + /** + * Returns the total requests per consumer. + * + * @return long total requests for consumer + */ + public long getTotalReqConsumer() { + return totalReqConsumer; + } + + /** + * Sets the total requests per consumer. + */ + public void incrementTotalReqConsumer() { + this.totalReqConsumer++; + } + + /** + * Sets the total failed requests per consumer. + */ + public void setTotalReqConsumer(Long totalReqConsumer) { + this.totalReqConsumer = totalReqConsumer; + } + + /** + * Returns the total fail requests per consumer. + * + * @return long total failed request for consumer + */ + public long getTotalFailReqConsumer() { + return totalFailReqConsumer; + } + + /** + * Sets the total fail requests per consumer. + */ + public void incrementTotalFailReqConsumer() { + this.totalFailReqConsumer++; + } + + /** + * Sets the total failed requests per consumer. + */ + public void setTotalFailReqConsumer(Long totalFailedReqConsumer) { + this.totalFailReqConsumer = totalFailedReqConsumer; + } + + /** + * Returns total requests per provider for operational level. + * + * @param key Operation path id + * @return long total requests per provider + */ + public Long getOperMetTotalReq(String key) { + return operMetricsTotalReq.get(key); + } + + /** + * Sets total requests per provider for operational level. + * + * @param key pathId + * @param val total requests per provider + */ + public void setOperMetTotalReq(String key, Long val) { + this.operMetricsTotalReq.put(key, val); + } + + /** + * Returns total fail requests per provider for operational level. + * + * @param key Operation path id + * @return long total fail requests per provider + */ + public Long getOperMetTotalFailReq(String key) { + return operMetricsTotalFailReq.get(key); + } + + /** + * Sets total fail requests per provider for operational level. + * + * @param key Operation path id + * @param val total fail requests per provider + */ + public void setOperMetTotalFailReq(String key, Long val) { + this.operMetricsTotalFailReq.put(key, val); + } +} diff --git a/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/MetricsDataMonitorUtil.java b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/MetricsDataMonitorUtil.java new file mode 100644 index 00000000000..8f16880d3ba --- /dev/null +++ b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/MetricsDataMonitorUtil.java @@ -0,0 +1,64 @@ +/* + * Copyright 2017 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.servicecomb.foundation.metrics.performance; + +import io.servicecomb.core.Invocation; +import io.servicecomb.foundation.metrics.MetricsServoRegistry; +import io.servicecomb.swagger.invocation.InvocationType; + +/** + * Implementation of metric util functions such as preparing average queue times + * and total requests/fail for producer and instance level. + */ +public final class MetricsDataMonitorUtil { + + /** + * Sets the total requests per provider and consumer. + * @param invocation invocation of request + */ + public void setAllReqProviderAndConsumer(Invocation invocation) { + MetricsDataMonitor metricsRef = MetricsServoRegistry.getOrCreateLocalMetrics(); + String operPath = invocation.getOperationMeta().getMicroserviceQualifiedName(); + + if (InvocationType.CONSUMER.equals(invocation.getInvocationType())) { + metricsRef.incrementTotalReqConsumer(); + } else { + metricsRef.incrementTotalReqProvider(); + // note down metrics for operational level. + metricsRef.setOperMetTotalReq(operPath, + metricsRef.getOperMetTotalReq(operPath) == null ? 1L : metricsRef.getOperMetTotalReq(operPath) + 1); + } + } + + /** + * Sets the total failed requests per provider and consumer. + * @param invocation invocation of request + */ + public void setAllFailReqProviderAndConsumer(Invocation invocation) { + MetricsDataMonitor metricsRef = MetricsServoRegistry.getOrCreateLocalMetrics(); + String operPath = invocation.getOperationMeta().getMicroserviceQualifiedName(); + + if (InvocationType.CONSUMER.equals(invocation.getInvocationType())) { + metricsRef.incrementTotalFailReqConsumer(); + } else { + metricsRef.incrementTotalFailReqProvider(); + // note down metrics for operational level. + metricsRef.setOperMetTotalReq(operPath, + metricsRef.getOperMetTotalReq(operPath) == null ? 1L : metricsRef.getOperMetTotalReq(operPath) + 1); + } + } +} diff --git a/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/QueueMetrics.java b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/QueueMetrics.java new file mode 100644 index 00000000000..613fee06e1f --- /dev/null +++ b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/QueueMetrics.java @@ -0,0 +1,97 @@ +/* + * Copyright 2017 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.servicecomb.foundation.metrics.performance; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * Used to store invocation specific metrics for queue. + */ +public class QueueMetrics { + + private AtomicLong queueStartTime = new AtomicLong(); + + private AtomicLong endOperTime = new AtomicLong(); + + private AtomicLong queueEndTime = new AtomicLong(); + + private String operQualifiedName; + + /** + * Returns the time when it enters the queue. + * @return long + */ + public long getQueueStartTime() { + return queueStartTime.get(); + } + + /** + * Sets the time when it enters the queue. + * @param startTime Entering time in queue + */ + public void setQueueStartTime(long startTime) { + this.queueStartTime.set(startTime); + } + + /** + * Returns the time when the operation ends. + * @return long + */ + public long getEndOperTime() { + return endOperTime.get(); + } + + /** + * Sets the time when the operation ends. + * @param stopOperTime Start time of operation + */ + public void setEndOperTime(long stopOperTime) { + this.endOperTime.set(stopOperTime); + } + + /** + * Returns the time when it leaves the queue. + * @return long + */ + public long getQueueEndTime() { + return queueEndTime.get(); + } + + /** + * Sets the time when it leaves the queue. + * @param endTime Leaving time from queue + */ + public void setQueueEndTime(long endTime) { + this.queueEndTime.set(endTime); + } + + /** + * Get the microservice qualified name. + * @return microservice qualified name + */ + public String getOperQualifiedName() { + return operQualifiedName; + } + + /** + * Set the microservice qualified name. + * @param operQualifiedName microservice qualified name + */ + public void setOperQualifiedName(String operQualifiedName) { + this.operQualifiedName = operQualifiedName; + } +} diff --git a/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/QueueMetricsData.java b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/QueueMetricsData.java new file mode 100644 index 00000000000..81a536afc47 --- /dev/null +++ b/foundations/foundation-metrics/src/main/java/io/servicecomb/foundation/metrics/performance/QueueMetricsData.java @@ -0,0 +1,216 @@ +/* + * Copyright 2017 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.servicecomb.foundation.metrics.performance; + +/** + * Used for holding the request queue related timings like start time, operation time and end time. + */ +public class QueueMetricsData { + + // after invocation polled from queue + private Long totalTime = new Long(0); + + // succ and fail + // after invocation polled from queue + private Long totalCount = new Long(0); + + // after invocation finished + private Long totalServExecutionTime = new Long(0); + + // after invocation finished + private Long totalServExecutionCount = new Long(0); + + // inc and dec + // addToQueue inc + // pollFromQueue inc + // pollFromQueue - addToQueue = countInQueue + // after invocation polled from queue + private Long countInQueue = new Long(0); + + // after invocation polled from queue + private Long minLifeTimeInQueue = new Long(0); + + // after invocation polled from queue + private Long maxLifeTimeInQueue = new Long(0); + + /** + * Sets total count in queue. + */ + public void incrementCountInQueue() { + this.countInQueue++; + } + + /** + * Deletes total count in queue. + */ + public void decrementCountInQueue() { + this.countInQueue--; + } + + /** + * default constructor. + */ + public QueueMetricsData() { + + } + + /** + * Returns the count for calculating average count value in queue. + * @return Long + */ + public Long getTotalTime() { + return totalTime; + } + + /** + * Sets the total time for calculating average time in queue. + * @param totalTime total time value + */ + public void setTotalTime(long totalTime) { + this.totalTime = totalTime; + } + + /** + * Returns the total count for calculating average count value in queue. + * @return Long + */ + public Long getTotalCount() { + return totalCount; + } + + /** + * Sets the total count for calculating average count value in queue. + */ + public void incrementTotalCount() { + this.totalCount++; + } + + /** + * Sets the total count for calculating average count value in queue. + * @param totalCount total count + */ + public void setTotalCount(long totalCount) { + this.totalCount = totalCount; + } + + /** + * Returns the count for calculating average value of working time after queue. + * @return Long + */ + public Long getTotalServExecutionTime() { + return totalServExecutionTime; + } + + /** + * Sets the count for calculating average value of working time after queue. + * @param totalCountAfterQueue count value + */ + public void setTotalServExecutionTime(long totalCountAfterQueue) { + this.totalServExecutionTime = totalCountAfterQueue; + } + + /** + * Returns the total count for calculating average value of working time after queue. + * @return Long + */ + public Long getTotalServExecutionCount() { + return totalServExecutionCount; + } + + /** + * Sets the total count for calculating average value of working time after queue. + */ + public void incrementTotalServExecutionCount() { + this.totalServExecutionCount++; + } + + /** + * Sets the total count for calculating average value of working time after queue. + * @param totalServExecutionCount total service execution time count + */ + public void setTotalServExecutionCount(long totalServExecutionCount) { + this.totalServExecutionCount = totalServExecutionCount; + } + + /** + * Returns total count in queue. + * @return Long + */ + public Long getCountInQueue() { + return countInQueue; + } + + /** + * Returns total count in queue. + * @param countInQueue queue count + */ + public void setCountInQueue(long countInQueue) { + this.countInQueue = countInQueue; + } + + /** + * Returns the minimum lifetime in queue. + * @return Long + */ + public Long getMinLifeTimeInQueue() { + return minLifeTimeInQueue; + } + + /** + * Sets the minimum lifetime in queue. + * @param minLifeTimeInQueue minimum lifetime + */ + public void setMinLifeTimeInQueue(long minLifeTimeInQueue) { + if ((this.minLifeTimeInQueue <= 0) || (minLifeTimeInQueue < this.minLifeTimeInQueue)) { + this.minLifeTimeInQueue = minLifeTimeInQueue; + } + } + + /** + * Returns maximum lifetime in queue. + * @return Long + */ + public Long getMaxLifeTimeInQueue() { + return maxLifeTimeInQueue; + } + + /** + * Sets maximum lifetime in queue. + * @param maxLifeTimeInQueue maximum lifetime + */ + public void setMaxLifeTimeInQueue(long maxLifeTimeInQueue) { + if ((this.maxLifeTimeInQueue <= 0) || (maxLifeTimeInQueue > this.maxLifeTimeInQueue)) { + this.maxLifeTimeInQueue = maxLifeTimeInQueue; + } + } + + /** + * Sets minimum lifetime in queue. + * @param maxLifeTimeInQueue maximum lifetime + */ + public void resetMinLifeTimeInQueue() { + this.minLifeTimeInQueue = 0L; + } + + /** + * resets maximum lifetime in queue. + * @param maxLifeTimeInQueue maximum lifetime + */ + public void resetMaxLifeTimeInQueue() { + this.maxLifeTimeInQueue = 0L; + } +} diff --git a/foundations/foundation-metrics/src/main/resources/META-INF/spring/metrics.bean.xml b/foundations/foundation-metrics/src/main/resources/META-INF/spring/metrics.bean.xml index 49b330acffa..3e70c7dc696 100644 --- a/foundations/foundation-metrics/src/main/resources/META-INF/spring/metrics.bean.xml +++ b/foundations/foundation-metrics/src/main/resources/META-INF/spring/metrics.bean.xml @@ -24,4 +24,6 @@ + + \ No newline at end of file diff --git a/foundations/foundation-metrics/src/test/java/io/servicecomb/foundation/metrics/TestMetricsServoRegistry.java b/foundations/foundation-metrics/src/test/java/io/servicecomb/foundation/metrics/TestMetricsServoRegistry.java new file mode 100644 index 00000000000..091722a5982 --- /dev/null +++ b/foundations/foundation-metrics/src/test/java/io/servicecomb/foundation/metrics/TestMetricsServoRegistry.java @@ -0,0 +1,168 @@ +/* + * Copyright 2017 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.servicecomb.foundation.metrics; + +import java.util.Map; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.netflix.servo.publish.PollScheduler; + +import io.servicecomb.foundation.metrics.performance.MetricsDataMonitor; +import io.servicecomb.foundation.metrics.performance.QueueMetricsData; + +public class TestMetricsServoRegistry { + MetricsDataMonitor metricsDataMonitor = null; + + MetricsDataMonitor localData = null; + + MetricsServoRegistry metricsRegistry = null; + + @Before + public void setUp() throws Exception { + metricsRegistry = new MetricsServoRegistry(); + localData = metricsRegistry.getLocalMetrics(); + metricsDataMonitor = MetricsServoRegistry.getOrCreateLocalMetrics(); + } + + @After + public void tearDown() throws Exception { + PollScheduler.getInstance().stop(); + metricsRegistry = null; + localData = null; + metricsDataMonitor = null; + } + + @Test + public void testAllRegistryMetrics() throws Exception { + metricsDataMonitor.incrementTotalReqProvider(); + metricsDataMonitor.incrementTotalFailReqProvider(); + metricsDataMonitor.incrementTotalReqConsumer(); + metricsDataMonitor.incrementTotalFailReqConsumer(); + metricsDataMonitor.setOperMetTotalReq("sayHi", 20L); + metricsDataMonitor.setOperMetTotalFailReq("sayHi", 20L); + localData = metricsRegistry.getLocalMetrics(); + localData.setOperMetTotalReq("sayHi", 10L); + localData.setOperMetTotalFailReq("sayHi", 10L); + + metricsRegistry.afterPropertiesSet(); + Thread.sleep(1000); + // get the metrics from local data and compare + localData = metricsRegistry.getLocalMetrics(); + Assert.assertEquals(1, localData.getTotalReqProvider()); + Assert.assertEquals(1, localData.getTotalFailReqProvider()); + Assert.assertEquals(1, localData.getTotalReqConsumer()); + Assert.assertEquals(1, localData.getTotalFailReqConsumer()); + Assert.assertEquals(20L, localData.getOperMetTotalReq("sayHi").longValue()); + Assert.assertEquals(20L, localData.getOperMetTotalFailReq("sayHi").longValue()); + + MetricsDataMonitor localData1 = metricsRegistry.getLocalMetrics(); + Assert.assertEquals(20L, localData1.getOperMetTotalReq("sayHi").longValue()); + Assert.assertEquals(20L, localData1.getOperMetTotalFailReq("sayHi").longValue()); + } + + @Test + public void testOperationalProviderMetrics() throws Exception { + MetricsDataMonitor metricsDataMonitor = MetricsServoRegistry.getOrCreateLocalMetrics(); + metricsDataMonitor.setOperMetTotalReq("sayHi", 20L); + metricsDataMonitor.setOperMetTotalFailReq("sayHi", 20L); + localData = metricsRegistry.getLocalMetrics(); + + metricsRegistry.afterPropertiesSet(); + Thread.sleep(1000); + // get the metrics from local data and compare + localData = metricsRegistry.getLocalMetrics(); + Assert.assertEquals(1, localData.getTotalReqProvider()); + Assert.assertEquals(1, localData.getTotalFailReqProvider()); + Assert.assertEquals(1, localData.getTotalReqConsumer()); + Assert.assertEquals(1, localData.getTotalFailReqConsumer()); + Assert.assertEquals(20L, localData.getOperMetTotalReq("sayHi").longValue()); + Assert.assertEquals(20L, localData.getOperMetTotalFailReq("sayHi").longValue()); + + MetricsDataMonitor localData1 = metricsRegistry.getLocalMetrics(); + Assert.assertEquals(20L, localData1.getOperMetTotalReq("sayHi").longValue()); + Assert.assertEquals(20L, localData1.getOperMetTotalFailReq("sayHi").longValue()); + } + + @Test + public void testQueueMetrics() throws Exception { + QueueMetricsData reqQueue1 = new QueueMetricsData(); + reqQueue1.setCountInQueue(1); + reqQueue1.setMaxLifeTimeInQueue(2); + reqQueue1.setMinLifeTimeInQueue(1); + reqQueue1.setTotalCount(10); + reqQueue1.setTotalTime(100); + reqQueue1.setTotalServExecutionCount(5); + reqQueue1.setTotalServExecutionTime(50); + metricsDataMonitor.setQueueMetrics("/sayBye", reqQueue1); + + metricsRegistry.afterPropertiesSet(); + Thread.sleep(1000); + // get the metrics from local data and compare + Map localMap = localData.getQueueMetrics(); + QueueMetricsData reqQueue2 = localMap.get("/sayBye"); + + Assert.assertEquals(1L, reqQueue2.getCountInQueue().longValue()); + Assert.assertEquals(1L, reqQueue2.getMinLifeTimeInQueue().longValue()); + Assert.assertEquals(2L, reqQueue2.getMaxLifeTimeInQueue().longValue()); + Assert.assertEquals(10L, reqQueue2.getTotalCount().longValue()); + Assert.assertEquals(100L, reqQueue2.getTotalTime().longValue()); + Assert.assertEquals(5, reqQueue2.getTotalServExecutionCount().longValue()); + Assert.assertEquals(50, reqQueue2.getTotalServExecutionTime().longValue()); + } + + @Test + public void testQueueMetrics1() throws Exception { + QueueMetricsData reqQueue1 = new QueueMetricsData(); + reqQueue1.setCountInQueue(10); + reqQueue1.setMaxLifeTimeInQueue(2); + reqQueue1.setMinLifeTimeInQueue(2); + reqQueue1.setTotalCount(1); + reqQueue1.setTotalTime(10); + reqQueue1.setTotalServExecutionCount(1); + reqQueue1.setTotalServExecutionTime(1); + localData.setQueueMetrics("/sayBye", reqQueue1); + + QueueMetricsData reqQueue2 = new QueueMetricsData(); + reqQueue2.setCountInQueue(10); + reqQueue2.setMaxLifeTimeInQueue(2); + reqQueue2.setMinLifeTimeInQueue(2); + reqQueue2.setTotalCount(10); + reqQueue2.setTotalTime(100); + reqQueue2.setTotalServExecutionCount(5); + reqQueue2.setTotalServExecutionTime(50); + metricsDataMonitor.setQueueMetrics("/sayBye", reqQueue2); + + metricsRegistry.afterPropertiesSet(); + Thread.sleep(1000); + // get the metrics from local data and compare + Map localMap = localData.getQueueMetrics(); + QueueMetricsData reqQueue3 = localMap.get("/sayBye"); + + Assert.assertEquals(10L, reqQueue3.getCountInQueue().longValue()); + Assert.assertEquals(2L, reqQueue3.getMinLifeTimeInQueue().longValue()); + Assert.assertEquals(2L, reqQueue3.getMaxLifeTimeInQueue().longValue()); + Assert.assertEquals(10L, reqQueue3.getTotalCount().longValue()); + Assert.assertEquals(100L, reqQueue3.getTotalTime().longValue()); + Assert.assertEquals(5L, reqQueue3.getTotalServExecutionCount().longValue()); + Assert.assertEquals(50L, reqQueue3.getTotalServExecutionTime().longValue()); + + } +} diff --git a/foundations/foundation-metrics/src/test/java/io/servicecomb/foundation/metrics/performance/TestMetricsDataMonitor.java b/foundations/foundation-metrics/src/test/java/io/servicecomb/foundation/metrics/performance/TestMetricsDataMonitor.java new file mode 100644 index 00000000000..2ef056bd35e --- /dev/null +++ b/foundations/foundation-metrics/src/test/java/io/servicecomb/foundation/metrics/performance/TestMetricsDataMonitor.java @@ -0,0 +1,131 @@ +/* + * Copyright 2017 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.servicecomb.foundation.metrics.performance; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestMetricsDataMonitor { + MetricsDataMonitor metricsDataMonitor = null; + + @Before + public void setUp() throws Exception { + metricsDataMonitor = new MetricsDataMonitor(); + } + + @After + public void tearDown() throws Exception { + metricsDataMonitor = null; + } + + @Test + public void testReqMapValues() { + QueueMetricsData reqQueue = new QueueMetricsData(); + reqQueue.setCountInQueue(1L); + reqQueue.setMaxLifeTimeInQueue(5L); + reqQueue.setMinLifeTimeInQueue(1L); + reqQueue.toString(); + + metricsDataMonitor.setQueueMetrics("/sayHi", reqQueue); + QueueMetricsData reqQueueTest = metricsDataMonitor.getOrCreateQueueMetrics("/sayHi"); + QueueMetricsData reqQueueTestAbsent = metricsDataMonitor.getOrCreateQueueMetrics(""); + Assert.assertEquals(1, reqQueueTest.getCountInQueue().longValue()); + Assert.assertEquals(5, reqQueueTest.getMaxLifeTimeInQueue().longValue()); + Assert.assertEquals(1, reqQueueTest.getMinLifeTimeInQueue().longValue()); + Assert.assertEquals(0, reqQueueTestAbsent.getCountInQueue().longValue()); + } + + @Test + public void testOperationMetricsMap() { + QueueMetrics reqQueue = new QueueMetrics(); + QueueMetricsData queueMetrics = new QueueMetricsData(); + queueMetrics.incrementCountInQueue(); + reqQueue.setQueueStartTime(200); + reqQueue.setEndOperTime(250); + reqQueue.setQueueEndTime(300); + reqQueue.setOperQualifiedName("name"); + queueMetrics.setTotalTime(100L); + queueMetrics.setTotalServExecutionTime(200L); + queueMetrics.setTotalCount(100L); + queueMetrics.setTotalServExecutionCount(200L); + + // increment the count to 5. + queueMetrics.incrementTotalCount(); + queueMetrics.incrementTotalCount(); + queueMetrics.incrementTotalCount(); + queueMetrics.incrementTotalCount(); + queueMetrics.incrementTotalCount(); + queueMetrics.incrementTotalServExecutionCount(); + queueMetrics.incrementTotalServExecutionCount(); + queueMetrics.incrementTotalServExecutionCount(); + queueMetrics.incrementTotalServExecutionCount(); + queueMetrics.incrementTotalServExecutionCount(); + + queueMetrics.setMinLifeTimeInQueue(1); + queueMetrics.resetMinLifeTimeInQueue(); + Assert.assertEquals(0, queueMetrics.getMinLifeTimeInQueue().longValue()); + + queueMetrics.setMaxLifeTimeInQueue(1); + queueMetrics.resetMaxLifeTimeInQueue(); + Assert.assertEquals(0, queueMetrics.getMaxLifeTimeInQueue().longValue()); + + metricsDataMonitor.setQueueMetrics("/sayHi", queueMetrics); + + //Assert.assertEquals(1, reqQueueTest.getConuntInQueue()); + Assert.assertEquals(300, reqQueue.getQueueEndTime()); + Assert.assertEquals(250, reqQueue.getEndOperTime()); + Assert.assertEquals(200, reqQueue.getQueueStartTime()); + Assert.assertEquals("name", reqQueue.getOperQualifiedName()); + Assert.assertEquals(100L, queueMetrics.getTotalTime().longValue()); + Assert.assertEquals(105L, queueMetrics.getTotalCount().longValue()); + Assert.assertEquals(200, queueMetrics.getTotalServExecutionTime().longValue()); + Assert.assertEquals(205L, queueMetrics.getTotalServExecutionCount().longValue()); + queueMetrics.decrementCountInQueue(); + Assert.assertEquals(0, queueMetrics.getCountInQueue().longValue()); + } + + @Test + public void testHystrixAvgTimes() { + + // total request for provider + metricsDataMonitor.incrementTotalReqProvider(); + metricsDataMonitor.incrementTotalFailReqProvider(); + Assert.assertEquals(1, metricsDataMonitor.getTotalReqProvider()); + Assert.assertEquals(1, metricsDataMonitor.getTotalFailReqProvider()); + metricsDataMonitor.incrementTotalReqProvider(); + metricsDataMonitor.incrementTotalFailReqProvider(); + Assert.assertEquals(2, metricsDataMonitor.getTotalReqProvider()); + Assert.assertEquals(2, metricsDataMonitor.getTotalFailReqProvider()); + + // total request for consumer + metricsDataMonitor.incrementTotalReqConsumer(); + metricsDataMonitor.incrementTotalFailReqConsumer(); + Assert.assertEquals(1, metricsDataMonitor.getTotalReqConsumer()); + Assert.assertEquals(1, metricsDataMonitor.getTotalFailReqConsumer()); + metricsDataMonitor.incrementTotalReqConsumer(); + metricsDataMonitor.incrementTotalFailReqConsumer(); + Assert.assertEquals(2, metricsDataMonitor.getTotalReqConsumer()); + Assert.assertEquals(2, metricsDataMonitor.getTotalFailReqConsumer()); + + metricsDataMonitor.setOperMetTotalReq("/sayHi", 10L); + metricsDataMonitor.setOperMetTotalFailReq("/sayHi", 20L); + Assert.assertEquals(10L, metricsDataMonitor.getOperMetTotalReq("/sayHi").longValue()); + Assert.assertEquals(20L, metricsDataMonitor.getOperMetTotalFailReq("/sayHi").longValue()); + } +} diff --git a/foundations/foundation-metrics/src/test/java/io/servicecomb/foundation/metrics/performance/TestMetricsDataMonitorUtil.java b/foundations/foundation-metrics/src/test/java/io/servicecomb/foundation/metrics/performance/TestMetricsDataMonitorUtil.java new file mode 100644 index 00000000000..e739194a57f --- /dev/null +++ b/foundations/foundation-metrics/src/test/java/io/servicecomb/foundation/metrics/performance/TestMetricsDataMonitorUtil.java @@ -0,0 +1,81 @@ +/* + * Copyright 2017 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.servicecomb.foundation.metrics.performance; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import io.servicecomb.core.Invocation; +import io.servicecomb.core.definition.OperationMeta; +import io.servicecomb.foundation.metrics.MetricsServoRegistry; +import io.servicecomb.swagger.invocation.InvocationType; + +public class TestMetricsDataMonitorUtil { + MetricsDataMonitor metricsDataMonitor = null; + + MetricsDataMonitorUtil metricsDataMonitorUtil = null; + + @Before + public void setUp() throws Exception { + metricsDataMonitor = MetricsServoRegistry.getOrCreateLocalMetrics(); + metricsDataMonitorUtil = new MetricsDataMonitorUtil(); + } + + @After + public void tearDown() throws Exception { + metricsDataMonitor = null; + } + + @Test + public void testAllReqConsumer() { + + Invocation invocation = Mockito.mock(Invocation.class); + OperationMeta operationMetaData = Mockito.mock(OperationMeta.class); + Mockito.when(invocation.getOperationMeta()).thenReturn(operationMetaData); + Mockito.when(operationMetaData.getMicroserviceQualifiedName()).thenReturn("/sayHi"); + Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("/sayHi"); + Mockito.when(invocation.getInvocationType()).thenReturn(InvocationType.CONSUMER); + + metricsDataMonitorUtil.setAllReqProviderAndConsumer(invocation); + Assert.assertNotEquals(0L, metricsDataMonitor.getTotalReqConsumer()); + metricsDataMonitorUtil.setAllFailReqProviderAndConsumer(invocation); + Assert.assertNotEquals(0L, metricsDataMonitor.getTotalFailReqConsumer()); + } + + @Test + public void testAllReqProvider() { + + Invocation invocation = Mockito.mock(Invocation.class); + OperationMeta operationMetaData = Mockito.mock(OperationMeta.class); + Mockito.when(invocation.getOperationMeta()).thenReturn(operationMetaData); + Mockito.when(operationMetaData.getMicroserviceQualifiedName()).thenReturn("/sayBye"); + Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("sayBye"); + Mockito.when(invocation.getInvocationType()).thenReturn(InvocationType.PRODUCER); + + metricsDataMonitorUtil.setAllReqProviderAndConsumer(invocation); + Assert.assertNotEquals(0L, metricsDataMonitor.getTotalReqProvider()); + metricsDataMonitorUtil.setAllReqProviderAndConsumer(invocation); + Assert.assertNotEquals(0L, metricsDataMonitor.getTotalReqProvider()); + metricsDataMonitorUtil.setAllFailReqProviderAndConsumer(invocation); + Assert.assertNotEquals(0L, metricsDataMonitor.getTotalFailReqProvider()); + metricsDataMonitorUtil.setAllFailReqProviderAndConsumer(invocation); + Assert.assertNotEquals(0L, metricsDataMonitor.getTotalFailReqProvider()); + } +} diff --git a/handlers/handler-bizkeeper/pom.xml b/handlers/handler-bizkeeper/pom.xml index f7d09b8844e..e471a06677f 100644 --- a/handlers/handler-bizkeeper/pom.xml +++ b/handlers/handler-bizkeeper/pom.xml @@ -50,7 +50,11 @@ log4j test - + + io.servicecomb + foundation-metrics + + diff --git a/handlers/handler-bizkeeper/src/main/java/io/servicecomb/bizkeeper/BizkeeperHandler.java b/handlers/handler-bizkeeper/src/main/java/io/servicecomb/bizkeeper/BizkeeperHandler.java index 7d532922e4a..4ab9f005389 100644 --- a/handlers/handler-bizkeeper/src/main/java/io/servicecomb/bizkeeper/BizkeeperHandler.java +++ b/handlers/handler-bizkeeper/src/main/java/io/servicecomb/bizkeeper/BizkeeperHandler.java @@ -28,6 +28,7 @@ import io.servicecomb.core.Handler; import io.servicecomb.core.Invocation; +import io.servicecomb.foundation.metrics.performance.MetricsDataMonitorUtil; import io.servicecomb.swagger.invocation.AsyncResponse; import io.servicecomb.swagger.invocation.Response; import rx.Observable; @@ -78,12 +79,17 @@ public BizkeeperHandler(String groupname) { public void handle(Invocation invocation, AsyncResponse asyncResp) throws Exception { HystrixObservable command = delegate.createBizkeeperCommand(invocation); + //Notedown all request for provider and consumer. + new MetricsDataMonitorUtil().setAllReqProviderAndConsumer(invocation); + Observable observable = command.toObservable(); observable.subscribe(response -> { asyncResp.complete(response); }, error -> { LOG.warn("catch error in bizkeeper:" + error.getMessage()); asyncResp.fail(invocation.getInvocationType(), error); + //Notedown all failed request for provider and consumer. + new MetricsDataMonitorUtil().setAllFailReqProviderAndConsumer(invocation); }, () -> { }); diff --git a/transports/transport-highway/pom.xml b/transports/transport-highway/pom.xml index be62c65c4b4..7c10563b720 100644 --- a/transports/transport-highway/pom.xml +++ b/transports/transport-highway/pom.xml @@ -46,5 +46,9 @@ log4j test + + io.servicecomb + foundation-metrics + diff --git a/transports/transport-highway/src/main/java/io/servicecomb/transport/highway/HighwayServerInvoke.java b/transports/transport-highway/src/main/java/io/servicecomb/transport/highway/HighwayServerInvoke.java index 405b22acc88..836d87c54db 100644 --- a/transports/transport-highway/src/main/java/io/servicecomb/transport/highway/HighwayServerInvoke.java +++ b/transports/transport-highway/src/main/java/io/servicecomb/transport/highway/HighwayServerInvoke.java @@ -32,6 +32,9 @@ import io.servicecomb.core.definition.MicroserviceMetaManager; import io.servicecomb.core.definition.OperationMeta; import io.servicecomb.core.definition.SchemaMeta; +import io.servicecomb.foundation.metrics.MetricsServoRegistry; +import io.servicecomb.foundation.metrics.performance.QueueMetrics; +import io.servicecomb.foundation.metrics.performance.QueueMetricsData; import io.servicecomb.foundation.vertx.tcp.TcpConnection; import io.servicecomb.swagger.invocation.Response; import io.servicecomb.swagger.invocation.exception.InvocationException; @@ -103,9 +106,9 @@ private void doInit(TcpConnection connection, long msgId, RequestHeader header, this.bodyBuffer = bodyBuffer; } - private void runInExecutor() { + private void runInExecutor(QueueMetrics metricsData) { try { - doRunInExecutor(); + doRunInExecutor(metricsData); } catch (Throwable e) { String msg = String.format("handle request error, %s, msgId=%d", operationMeta.getMicroserviceQualifiedName(), @@ -116,11 +119,13 @@ private void runInExecutor() { } } - private void doRunInExecutor() throws Exception { + private void doRunInExecutor(QueueMetrics metricsData) throws Exception { Invocation invocation = HighwayCodec.decodeRequest(header, operationProtobuf, bodyBuffer, protobufFeature); invocation.getHandlerContext().put(Const.REMOTE_ADDRESS, this.connection.getNetSocket().remoteAddress()); + updateMetrics(invocation); invocation.next(response -> { sendResponse(invocation.getContext(), response); + endMetrics(invocation); }); } @@ -149,7 +154,59 @@ private void sendResponse(Map context, Response response) { } } + /** + * start time in queue. + */ public void execute() { - operationMeta.getExecutor().execute(this::runInExecutor); + QueueMetrics metricsData = initMetrics(operationMeta); + operationMeta.getExecutor().execute(() -> runInExecutor(metricsData)); + } + + /** + * Init the metrics. Note down the queue count and start time. + * @param operationMeta Operation data + * @return QueueMetrics + */ + private QueueMetrics initMetrics(OperationMeta operationMeta) { + QueueMetrics metricsData = new QueueMetrics(); + metricsData.setQueueStartTime(System.currentTimeMillis()); + metricsData.setOperQualifiedName(operationMeta.getMicroserviceQualifiedName()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(operationMeta.getMicroserviceQualifiedName()); + reqQueue.incrementCountInQueue(); + return metricsData; + } + + /** + * Update the queue metrics. + */ + private void updateMetrics(Invocation invocation) { + QueueMetrics metricsData = (QueueMetrics) invocation.getMetricsData(); + if (null != metricsData) { + metricsData.setQueueEndTime(System.currentTimeMillis()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(operationMeta.getMicroserviceQualifiedName()); + reqQueue.incrementTotalCount(); + Long timeInQueue = metricsData.getQueueEndTime() - metricsData.getQueueStartTime(); + reqQueue.setTotalTime(reqQueue.getTotalTime() + timeInQueue); + reqQueue.setMinLifeTimeInQueue(timeInQueue); + reqQueue.setMaxLifeTimeInQueue(timeInQueue); + reqQueue.decrementCountInQueue(); + } + } + + /** + * Prepare the end time of queue metrics. + */ + private void endMetrics(Invocation invocation) { + QueueMetrics metricsData = (QueueMetrics) invocation.getMetricsData(); + if (null != metricsData) { + metricsData.setEndOperTime(System.currentTimeMillis()); + QueueMetricsData reqQueue = MetricsServoRegistry.getOrCreateLocalMetrics() + .getOrCreateQueueMetrics(operationMeta.getMicroserviceQualifiedName()); + reqQueue.incrementTotalServExecutionCount(); + reqQueue.setTotalServExecutionTime( + reqQueue.getTotalServExecutionTime() + (metricsData.getEndOperTime() - metricsData.getQueueEndTime())); + } } } diff --git a/transports/transport-highway/src/test/java/io/servicecomb/transport/highway/TestHighwayServerInvoke.java b/transports/transport-highway/src/test/java/io/servicecomb/transport/highway/TestHighwayServerInvoke.java index 5e4c68252fa..7f83739bd1d 100644 --- a/transports/transport-highway/src/test/java/io/servicecomb/transport/highway/TestHighwayServerInvoke.java +++ b/transports/transport-highway/src/test/java/io/servicecomb/transport/highway/TestHighwayServerInvoke.java @@ -107,11 +107,6 @@ public void test() { requestHeader.setOperationName(operationMeta.getOperationId()); Assert.assertTrue(highwayServerInvoke.init(connection, 0, requestHeader, null)); - // exe成功 - netSocketBuffer = null; - highwayServerInvoke.execute(); - Assert.assertEquals(null, netSocketBuffer); - // exe失败 MockUtil.getInstance().decodeRequestSucc = false; highwayServerInvoke.execute();