Observability task: metadata center (apache#11593)

* init metadata * add pom * add licence * add licence * remove unuse pom * remove unuse pom * remove unuse pom * fix test * fix test * fix test * fix pom * use applicationModel * remove unuse * add test * add push testcase * add test case * add test case * add testcase * rename * opt * debug * fix testcase * add pom --------- Co-authored-by: x-shadow-man <1494445739@qq.com>
mytang0 · Feb 22, 2023 · 671e0f8 · 671e0f8
1 parent 2897c85
commit 671e0f8
Show file tree

Hide file tree

Showing 28 changed files with 919 additions and 43 deletions.
diff --git a/dubbo-common/src/main/java/org/apache/dubbo/config/MetricsConfig.java b/dubbo-common/src/main/java/org/apache/dubbo/config/MetricsConfig.java
@@ -16,16 +16,16 @@
  */
 package org.apache.dubbo.config;
 
-import java.util.HashMap;
-import java.util.Map;
-
 import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.utils.UrlUtils;
 import org.apache.dubbo.config.nested.AggregationConfig;
 import org.apache.dubbo.config.nested.PrometheusConfig;
 import org.apache.dubbo.config.support.Nested;
 import org.apache.dubbo.rpc.model.ApplicationModel;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * MetricsConfig
  */
@@ -40,6 +40,11 @@ public class MetricsConfig extends AbstractConfig {
      */
     private Boolean enableJvmMetrics;
 
+    /**
+     * Enable jvm metrics when collecting.
+     */
+    private Boolean enableMetadataMetrics;
+
     /**
      * @deprecated After metrics config is refactored.
      * This parameter should no longer use and will be deleted in the future.
@@ -137,5 +142,13 @@ public Integer getExportServicePort() {
     public void setExportServicePort(Integer exportServicePort) {
         this.exportServicePort = exportServicePort;
     }
+
+    public Boolean getEnableMetadataMetrics() {
+        return enableMetadataMetrics;
+    }
+
+    public void setEnableMetadataMetrics(Boolean enableMetadataMetrics) {
+        this.enableMetadataMetrics = enableMetadataMetrics;
+    }
 }
 
diff --git a/dubbo-config/dubbo-config-api/pom.xml b/dubbo-config/dubbo-config-api/pom.xml
@@ -48,6 +48,12 @@
             <version>${project.parent.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-metrics-metadata</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.dubbo</groupId>
             <artifactId>dubbo-monitor-api</artifactId>

diff --git a/...o-config-api/src/main/java/org/apache/dubbo/config/deploy/DefaultApplicationDeployer.java b/...o-config-api/src/main/java/org/apache/dubbo/config/deploy/DefaultApplicationDeployer.java
@@ -359,7 +359,7 @@ private void initMetricsService() {
 
     private void initMetricsReporter() {
         DefaultMetricsCollector collector =
-            applicationModel.getFrameworkModel().getBeanFactory().getBean(DefaultMetricsCollector.class);
+            applicationModel.getBeanFactory().getBean(DefaultMetricsCollector.class);
         MetricsConfig metricsConfig = configManager.getMetrics().orElse(null);
         // TODO compatible with old usage of metrics, remove protocol check after new metrics is ready for use.
         if (metricsConfig != null && PROTOCOL_PROMETHEUS.equals(metricsConfig.getProtocol())) {

diff --git a/dubbo-distribution/dubbo-all/pom.xml b/dubbo-distribution/dubbo-all/pom.xml
@@ -197,6 +197,13 @@
             <scope>compile</scope>
             <optional>true</optional>
         </dependency>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-metrics-metadata</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+            <optional>true</optional>
+        </dependency>
 
         <!-- monitor -->
         <dependency>
@@ -495,6 +502,7 @@
                                     <include>org.apache.dubbo:dubbo-metadata-report-zookeeper</include>
                                     <include>org.apache.dubbo:dubbo-metrics-api</include>
                                     <include>org.apache.dubbo:dubbo-metrics-default</include>
+                                    <include>org.apache.dubbo:dubbo-metrics-metadata</include>
                                     <include>org.apache.dubbo:dubbo-metrics-prometheus</include>
                                     <include>org.apache.dubbo:dubbo-monitor-api</include>
                                     <include>org.apache.dubbo:dubbo-monitor-default</include>

diff --git a/dubbo-distribution/dubbo-bom/pom.xml b/dubbo-distribution/dubbo-bom/pom.xml
@@ -240,6 +240,11 @@
                 <artifactId>dubbo-metrics-prometheus</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.dubbo</groupId>
+                <artifactId>dubbo-metrics-metadata</artifactId>
+                <version>${project.version}</version>
+            </dependency>
 
             <!-- monitor -->
             <dependency>

diff --git a/...ics-api/src/main/java/org/apache/dubbo/metrics/collector/ApplicationMetricsCollector.java b/...ics-api/src/main/java/org/apache/dubbo/metrics/collector/ApplicationMetricsCollector.java
@@ -17,18 +17,18 @@
 
 package org.apache.dubbo.metrics.collector;
 
+import org.apache.dubbo.metrics.event.MetricsEvent;
+
 /**
  * Application-level collector.
  * registration center, configuration center and other scenarios
  *
  * @Params <T>  metrics type
  */
-public interface ApplicationMetricsCollector<T> extends MetricsCollector {
+public interface ApplicationMetricsCollector<T, E extends MetricsEvent> extends MetricsCollector<E> {
 
     void increment(String applicationName, T type);
 
-    void decrease(String applicationName, T type);
-
-    void addRT(String applicationName, Long responseTime);
+    void addRT(String applicationName, String registryOpType, Long responseTime);
 }
 
diff --git a/.../dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/collector/MetricsCollector.java b/.../dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/collector/MetricsCollector.java
@@ -19,7 +19,7 @@
 
 import org.apache.dubbo.common.extension.SPI;
 import org.apache.dubbo.metrics.event.MetricsEvent;
-import org.apache.dubbo.metrics.listener.MetricsListener;
+import org.apache.dubbo.metrics.listener.MetricsLifeListener;
 import org.apache.dubbo.metrics.model.sample.MetricSample;
 
 import java.util.List;
@@ -29,20 +29,17 @@
  * An interface of collector to collect framework internal metrics.
  */
 @SPI
-public interface MetricsCollector extends MetricsListener {
+public interface MetricsCollector<E extends MetricsEvent> extends MetricsLifeListener<E> {
 
     default boolean isCollectEnabled() {
         return false;
     }
+
     /**
      * Collect metrics as {@link MetricSample}
      *
      * @return List of MetricSample
      */
     List<MetricSample> collect();
 
-    @Override
-    default void onEvent(MetricsEvent event) {
-
-    }
 }
diff --git a/...trics-api/src/main/java/org/apache/dubbo/metrics/event/GlobalMetricsEventMulticaster.java b/...trics-api/src/main/java/org/apache/dubbo/metrics/event/GlobalMetricsEventMulticaster.java
@@ -20,15 +20,19 @@
 import org.apache.dubbo.common.beans.factory.ScopeBeanFactory;
 import org.apache.dubbo.common.extension.ExtensionLoader;
 import org.apache.dubbo.metrics.collector.MetricsCollector;
-import org.apache.dubbo.rpc.model.FrameworkModel;
+import org.apache.dubbo.rpc.model.ApplicationModel;
 
 import java.util.List;
 
+/**
+ * Global spi event publisher
+ */
 public class GlobalMetricsEventMulticaster extends SimpleMetricsEventMulticaster {
 
-    public GlobalMetricsEventMulticaster(FrameworkModel frameworkModel) {
-        ScopeBeanFactory beanFactory = frameworkModel.getBeanFactory();
-        ExtensionLoader<MetricsCollector> extensionLoader = frameworkModel.getExtensionLoader(MetricsCollector.class);
+    @SuppressWarnings({"rawtypes"})
+    public GlobalMetricsEventMulticaster(ApplicationModel applicationModel) {
+        ScopeBeanFactory beanFactory = applicationModel.getBeanFactory();
+        ExtensionLoader<MetricsCollector> extensionLoader = applicationModel.getExtensionLoader(MetricsCollector.class);
         if (extensionLoader != null) {
             List<MetricsCollector> customizeCollectors = extensionLoader
                 .getActivateExtensions();

diff --git a/...ubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/listener/MetricsLifeListener.java b/...ubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/listener/MetricsLifeListener.java
@@ -24,7 +24,9 @@
  */
 public interface MetricsLifeListener<E extends MetricsEvent> extends MetricsListener<E> {
 
-    void onEventFinish(E event);
+    default void onEventFinish(E event) {
+    }
 
-    void onEventError(E event);
+    default void onEventError(E event) {
+    }
 }
diff --git a/...cs/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/listener/MetricsListener.java b/...cs/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/listener/MetricsListener.java
@@ -33,6 +33,7 @@ default boolean isSupport(MetricsEvent event) {
      *
      * @param event BaseMetricsEvent
      */
-    void onEvent(E event);
+    default void onEvent(E event) {
+    }
 
 }
diff --git a/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/MetricsKey.java b/dubbo-metrics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/MetricsKey.java
@@ -30,7 +30,6 @@ public enum MetricsKey {
     METRIC_REQUESTS_FAILED("dubbo.%s.requests.unknown.failed.total", "Unknown Failed Requests"),
     METRIC_REQUESTS_TOTAL_FAILED("dubbo.%s.requests.failed.total", "Total Failed Requests"),
 
-
     METRIC_REQUESTS_TOTAL_AGG("dubbo.%s.requests.total.aggregate", "Aggregated Total Requests"),
     METRIC_REQUESTS_SUCCEED_AGG("dubbo.%s.requests.succeed.aggregate", "Aggregated Succeed Requests"),
     METRIC_REQUESTS_FAILED_AGG("dubbo.%s.requests.failed.aggregate", "Aggregated Failed Requests"),
@@ -60,12 +59,25 @@ public enum MetricsKey {
     GENERIC_METRIC_RT_P99("dubbo.%s.rt.seconds.p99", "Response Time P99"),
     GENERIC_METRIC_RT_P95("dubbo.%s.rt.seconds.p95", "Response Time P95"),
 
-    THREAD_POOL_CORE_SIZE("dubbo.thread.pool.core.size","Thread Pool Core Size"),
-    THREAD_POOL_LARGEST_SIZE("dubbo.thread.pool.largest.size","Thread Pool Largest Size"),
-    THREAD_POOL_MAX_SIZE("dubbo.thread.pool.max.size","Thread Pool Max Size"),
-    THREAD_POOL_ACTIVE_SIZE("dubbo.thread.pool.active.size","Thread Pool Active Size"),
-    THREAD_POOL_THREAD_COUNT("dubbo.thread.pool.thread.count","Thread Pool Thread Count"),
-    THREAD_POOL_QUEUE_SIZE("dubbo.thread.pool.queue.size","Thread Pool Queue Size");
+    THREAD_POOL_CORE_SIZE("dubbo.thread.pool.core.size", "Thread Pool Core Size"),
+    THREAD_POOL_LARGEST_SIZE("dubbo.thread.pool.largest.size", "Thread Pool Largest Size"),
+    THREAD_POOL_MAX_SIZE("dubbo.thread.pool.max.size", "Thread Pool Max Size"),
+    THREAD_POOL_ACTIVE_SIZE("dubbo.thread.pool.active.size", "Thread Pool Active Size"),
+    THREAD_POOL_THREAD_COUNT("dubbo.thread.pool.thread.count", "Thread Pool Thread Count"),
+    THREAD_POOL_QUEUE_SIZE("dubbo.thread.pool.queue.size", "Thread Pool Queue Size"),
+
+    // metadata push metrics key
+    METADATA_PUSH_METRIC_NUM("dubbo.metadata.push.num.total", "Total Push Num"),
+    METADATA_PUSH_METRIC_NUM_SUCCEED("dubbo.metadata.push.num.succeed.total", "Succeed Push Num"),
+    METADATA_PUSH_METRIC_NUM_FAILED("dubbo.metadata.push.num.failed.total", "Failed Push Num"),
+
+    // metadata subscribe metrics key
+    METADATA_SUBSCRIBE_METRIC_NUM("dubbo.metadata.subscribe.num.total", "Total Metadata Subscribe Num"),
+    METADATA_SUBSCRIBE_METRIC_NUM_SUCCEED("dubbo.metadata.subscribe.num.succeed.total", "Succeed Metadata Subscribe Num"),
+    METADATA_SUBSCRIBE_METRIC_NUM_FAILED("dubbo.metadata.subscribe.num.failed.total", "Failed Metadata Subscribe Num"),
+
+    // consumer metrics key
+    ;
 
     private final String name;
     private final String description;

diff --git a/...ics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/MetricsKeyWrapper.java b/...ics/dubbo-metrics-api/src/main/java/org/apache/dubbo/metrics/model/MetricsKeyWrapper.java
@@ -38,8 +38,8 @@ public MetricsKey getMetricsKey() {
         return metricsKey;
     }
 
-    public boolean isKey(MetricsKey metricsKey) {
-        return metricsKey == getMetricsKey();
+    public boolean isKey(MetricsKey metricsKey, String registryOpType) {
+        return metricsKey == getMetricsKey() && registryOpType.equals(getType());
     }
 
     public String targetKey() {

diff --git a/...s-api/src/test/java/org/apache/dubbo/metrics/event/SimpleMetricsEventMulticasterTest.java b/...s-api/src/test/java/org/apache/dubbo/metrics/event/SimpleMetricsEventMulticasterTest.java
@@ -18,6 +18,7 @@
 package org.apache.dubbo.metrics.event;
 
 import org.apache.dubbo.metrics.listener.MetricsLifeListener;
+import org.apache.dubbo.metrics.listener.MetricsListener;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -32,7 +33,12 @@ public class SimpleMetricsEventMulticasterTest {
     public void setup() {
         eventMulticaster = new SimpleMetricsEventMulticaster();
         obj = new Object[]{new Object()};
-        eventMulticaster.addListener(event -> obj[0] = new Object());
+        eventMulticaster.addListener(new MetricsListener<MetricsEvent>() {
+            @Override
+            public void onEvent(MetricsEvent event) {
+                obj[0] = new Object();
+            }
+        });
         requestEvent = new RequestEvent(obj[0], MetricsEvent.Type.TOTAL);
 
     }

diff --git a/...-metrics-default/src/main/java/org/apache/dubbo/metrics/MetricsScopeModelInitializer.java b/...-metrics-default/src/main/java/org/apache/dubbo/metrics/MetricsScopeModelInitializer.java
@@ -34,7 +34,7 @@ public void initializeFrameworkModel(FrameworkModel frameworkModel) {
 
     @Override
     public void initializeApplicationModel(ApplicationModel applicationModel) {
-        ScopeBeanFactory beanFactory = applicationModel.getFrameworkModel().getBeanFactory();
+        ScopeBeanFactory beanFactory = applicationModel.getBeanFactory();
         beanFactory.registerBean(DefaultMetricsCollector.class);
         beanFactory.registerBean(GlobalMetricsEventMulticaster.class);
     }

diff --git a/dubbo-metrics/dubbo-metrics-metadata/pom.xml b/dubbo-metrics/dubbo-metrics-metadata/pom.xml
@@ -0,0 +1,40 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.dubbo</groupId>
+        <artifactId>dubbo-metrics</artifactId>
+        <version>${revision}</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <artifactId>dubbo-metrics-metadata</artifactId>
+    <packaging>jar</packaging>
+    <name>${project.artifactId}</name>
+    <description>The metrics module of dubbo project</description>
+    <properties>
+        <skip_maven_deploy>false</skip_maven_deploy>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-metrics-api</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+    </dependencies>
+</project>