Refactor Distro Config as singleton and replace GlobalConfig (#5486)

* Refactor Distro Config as singleton and replace GlobalConfig * Use configed timeout for distro sync and verify * Fix IT error.
alibaba · Apr 25, 2021 · 71a0037 · 71a0037
1 parent 267d671
commit 71a0037
Show file tree

Hide file tree

Showing 12 changed files with 204 additions and 105 deletions.
diff --git a/core/src/main/java/com/alibaba/nacos/core/distributed/distro/DistroConfig.java b/core/src/main/java/com/alibaba/nacos/core/distributed/distro/DistroConfig.java
@@ -16,28 +16,56 @@
 
 package com.alibaba.nacos.core.distributed.distro;
 
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
+import com.alibaba.nacos.core.utils.Loggers;
+import com.alibaba.nacos.sys.env.EnvUtil;
 
 /**
  * Distro configuration.
  *
  * @author xiweng.yy
  */
-@Component
 public class DistroConfig {
 
-    @Value("${nacos.core.protocol.distro.data.sync_delay_ms:1000}")
-    private long syncDelayMillis = 1000;
+    private static final DistroConfig INSTANCE = new DistroConfig();
 
-    @Value("${nacos.core.protocol.distro.data.sync_retry_delay_ms:3000}")
-    private long syncRetryDelayMillis = 3000;
+    private long syncDelayMillis = DistroConstants.DEFAULT_DATA_SYNC_DELAY_MILLISECONDS;
 
-    @Value("${nacos.core.protocol.distro.data.verify_interval_ms:5000}")
-    private long verifyIntervalMillis = 5000;
+    private long syncTimeoutMillis = DistroConstants.DEFAULT_DATA_SYNC_TIMEOUT_MILLISECONDS;
 
-    @Value("${nacos.core.protocol.distro.data.load_retry_delay_ms:30000}")
-    private long loadDataRetryDelayMillis = 30000;
+    private long syncRetryDelayMillis = DistroConstants.DEFAULT_DATA_SYNC_RETRY_DELAY_MILLISECONDS;
+
+    private long verifyIntervalMillis = DistroConstants.DEFAULT_DATA_VERIFY_INTERVAL_MILLISECONDS;
+
+    private long verifyTimeoutMillis = DistroConstants.DEFAULT_DATA_VERIFY_TIMEOUT_MILLISECONDS;
+
+    private long loadDataRetryDelayMillis = DistroConstants.DEFAULT_DATA_LOAD_RETRY_DELAY_MILLISECONDS;
+
+    private DistroConfig() {
+        try {
+            getDistroConfigFromEnv();
+        } catch (Exception e) {
+            Loggers.CORE.warn("Get Distro config from env failed, will use default value", e);
+        }
+    }
+
+    private void getDistroConfigFromEnv() {
+        syncDelayMillis = EnvUtil.getProperty(DistroConstants.DATA_SYNC_DELAY_MILLISECONDS, Long.class,
+                DistroConstants.DEFAULT_DATA_SYNC_DELAY_MILLISECONDS);
+        syncTimeoutMillis = EnvUtil.getProperty(DistroConstants.DATA_SYNC_TIMEOUT_MILLISECONDS, Long.class,
+                DistroConstants.DEFAULT_DATA_SYNC_TIMEOUT_MILLISECONDS);
+        syncRetryDelayMillis = EnvUtil.getProperty(DistroConstants.DATA_SYNC_RETRY_DELAY_MILLISECONDS, Long.class,
+                DistroConstants.DEFAULT_DATA_SYNC_RETRY_DELAY_MILLISECONDS);
+        verifyIntervalMillis = EnvUtil.getProperty(DistroConstants.DATA_VERIFY_INTERVAL_MILLISECONDS, Long.class,
+                DistroConstants.DEFAULT_DATA_VERIFY_INTERVAL_MILLISECONDS);
+        verifyTimeoutMillis = EnvUtil.getProperty(DistroConstants.DATA_VERIFY_TIMEOUT_MILLISECONDS, Long.class,
+                DistroConstants.DEFAULT_DATA_VERIFY_TIMEOUT_MILLISECONDS);
+        loadDataRetryDelayMillis = EnvUtil.getProperty(DistroConstants.DATA_LOAD_RETRY_DELAY_MILLISECONDS, Long.class,
+                DistroConstants.DEFAULT_DATA_LOAD_RETRY_DELAY_MILLISECONDS);
+    }
+
+    public static DistroConfig getInstance() {
+        return INSTANCE;
+    }
 
     public long getSyncDelayMillis() {
         return syncDelayMillis;
@@ -47,6 +75,14 @@ public void setSyncDelayMillis(long syncDelayMillis) {
         this.syncDelayMillis = syncDelayMillis;
     }
 
+    public long getSyncTimeoutMillis() {
+        return syncTimeoutMillis;
+    }
+
+    public void setSyncTimeoutMillis(long syncTimeoutMillis) {
+        this.syncTimeoutMillis = syncTimeoutMillis;
+    }
+
     public long getSyncRetryDelayMillis() {
         return syncRetryDelayMillis;
     }
@@ -63,6 +99,14 @@ public void setVerifyIntervalMillis(long verifyIntervalMillis) {
         this.verifyIntervalMillis = verifyIntervalMillis;
     }
 
+    public long getVerifyTimeoutMillis() {
+        return verifyTimeoutMillis;
+    }
+
+    public void setVerifyTimeoutMillis(long verifyTimeoutMillis) {
+        this.verifyTimeoutMillis = verifyTimeoutMillis;
+    }
+
     public long getLoadDataRetryDelayMillis() {
         return loadDataRetryDelayMillis;
     }

diff --git a/core/src/main/java/com/alibaba/nacos/core/distributed/distro/DistroConstants.java b/core/src/main/java/com/alibaba/nacos/core/distributed/distro/DistroConstants.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 1999-2020 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.alibaba.nacos.core.distributed.distro;
+
+/**
+ * Distro constants.
+ *
+ * @author xiweng.yy
+ */
+public class DistroConstants {
+
+    public static final String DATA_SYNC_DELAY_MILLISECONDS = "nacos.core.protocol.distro.data.sync.delayMs";
+
+    public static final long DEFAULT_DATA_SYNC_DELAY_MILLISECONDS = 1000L;
+
+    public static final String DATA_SYNC_TIMEOUT_MILLISECONDS = "nacos.core.protocol.distro.data.sync.timeoutMs";
+
+    public static final long DEFAULT_DATA_SYNC_TIMEOUT_MILLISECONDS = 3000L;
+
+    public static final String DATA_SYNC_RETRY_DELAY_MILLISECONDS = "nacos.core.protocol.distro.data.sync.retryDelayMs";
+
+    public static final long DEFAULT_DATA_SYNC_RETRY_DELAY_MILLISECONDS = 3000L;
+
+    public static final String DATA_VERIFY_INTERVAL_MILLISECONDS = "nacos.core.protocol.distro.data.verify.intervalMs";
+
+    public static final long DEFAULT_DATA_VERIFY_INTERVAL_MILLISECONDS = 5000L;
+
+    public static final String DATA_VERIFY_TIMEOUT_MILLISECONDS = "nacos.core.protocol.distro.data.verify.timeoutMs";
+
+    public static final long DEFAULT_DATA_VERIFY_TIMEOUT_MILLISECONDS = 3000L;
+
+    public static final String DATA_LOAD_RETRY_DELAY_MILLISECONDS = "nacos.core.protocol.distro.data.load.retryDelayMs";
+
+    public static final long DEFAULT_DATA_LOAD_RETRY_DELAY_MILLISECONDS = 30000L;
+
+}
diff --git a/core/src/main/java/com/alibaba/nacos/core/distributed/distro/DistroProtocol.java b/core/src/main/java/com/alibaba/nacos/core/distributed/distro/DistroProtocol.java
@@ -49,16 +49,13 @@ public class DistroProtocol {
 
     private final DistroTaskEngineHolder distroTaskEngineHolder;
 
-    private final DistroConfig distroConfig;
-
     private volatile boolean isInitialized = false;
 
     public DistroProtocol(ServerMemberManager memberManager, DistroComponentHolder distroComponentHolder,
-            DistroTaskEngineHolder distroTaskEngineHolder, DistroConfig distroConfig) {
+            DistroTaskEngineHolder distroTaskEngineHolder) {
         this.memberManager = memberManager;
         this.distroComponentHolder = distroComponentHolder;
         this.distroTaskEngineHolder = distroTaskEngineHolder;
-        this.distroConfig = distroConfig;
         startDistroTask();
     }
 
@@ -84,12 +81,13 @@ public void onFailed(Throwable throwable) {
             }
         };
         GlobalExecutor.submitLoadDataTask(
-                new DistroLoadDataTask(memberManager, distroComponentHolder, distroConfig, loadCallback));
+                new DistroLoadDataTask(memberManager, distroComponentHolder, DistroConfig.getInstance(), loadCallback));
     }
 
     private void startVerifyTask() {
         GlobalExecutor.schedulePartitionDataTimedSync(new DistroVerifyTimedTask(memberManager, distroComponentHolder,
-                distroTaskEngineHolder.getExecuteWorkersManager()), distroConfig.getVerifyIntervalMillis());
+                        distroTaskEngineHolder.getExecuteWorkersManager()),
+                DistroConfig.getInstance().getVerifyIntervalMillis());
     }
 
     public boolean isInitialized() {
@@ -103,7 +101,7 @@ public boolean isInitialized() {
      * @param action    the action of data operation
      */
     public void sync(DistroKey distroKey, DataOperation action) {
-        sync(distroKey, action, distroConfig.getSyncDelayMillis());
+        sync(distroKey, action, DistroConfig.getInstance().getSyncDelayMillis());
     }
 
     /**
@@ -164,7 +162,8 @@ public DistroData queryFromRemote(DistroKey distroKey) {
      * @return true if handle receive data successfully, otherwise false
      */
     public boolean onReceive(DistroData distroData) {
-        Loggers.DISTRO.info("[DISTRO] Receive distro data type: {}, key: {}", distroData.getType(), distroData.getDistroKey());
+        Loggers.DISTRO.info("[DISTRO] Receive distro data type: {}, key: {}", distroData.getType(),
+                distroData.getDistroKey());
         String resourceType = distroData.getDistroKey().getResourceType();
         DistroDataProcessor dataProcessor = distroComponentHolder.findDataProcessor(resourceType);
         if (null == dataProcessor) {
@@ -183,7 +182,8 @@ public boolean onReceive(DistroData distroData) {
      */
     public boolean onVerify(DistroData distroData, String sourceAddress) {
         if (Loggers.DISTRO.isDebugEnabled()) {
-            Loggers.DISTRO.debug("[DISTRO] Receive verify data type: {}, key: {}", distroData.getType(), distroData.getDistroKey());
+            Loggers.DISTRO.debug("[DISTRO] Receive verify data type: {}, key: {}", distroData.getType(),
+                    distroData.getDistroKey());
         }
         String resourceType = distroData.getDistroKey().getResourceType();
         DistroDataProcessor dataProcessor = distroComponentHolder.findDataProcessor(resourceType);

diff --git a/distribution/conf/application.properties b/distribution/conf/application.properties
@@ -18,7 +18,6 @@
 ### Default web context path:
 server.servlet.contextPath=/nacos
 ### Default web server port:
-server.port=8848
 
 #*************** Network Related Configurations ***************#
 ### If prefer hostname over ip for Nacos server addresses in cluster.conf:
@@ -40,20 +39,14 @@ server.port=8848
 # db.user.0=nacos
 # db.password.0=nacos
 
-### Connection pool configuration: hikariCP
-db.pool.config.connectionTimeout=30000
-db.pool.config.validationTimeout=10000
-db.pool.config.maximumPoolSize=20
-db.pool.config.minimumIdle=2
-
 #*************** Naming Module Related Configurations ***************#
-### Data dispatch task execution period in milliseconds:
+### Data dispatch task execution period in milliseconds: Will removed on v2.1.X, replace with nacos.core.protocol.distro.data.sync.delayMs
 # nacos.naming.distro.taskDispatchPeriod=200
 
-### Data count of batch sync task:
+### Data count of batch sync task: Will removed on v2.1.X. Deprecated
 # nacos.naming.distro.batchSyncKeyCount=1000
 
-### Retry delay in milliseconds if sync task failed:
+### Retry delay in milliseconds if sync task failed: Will removed on v2.1.X, replace with nacos.core.protocol.distro.data.sync.retryDelayMs
 # nacos.naming.distro.syncRetryDelay=5000
 
 ### If enable data warmup. If set to false, the server would accept request without local data preparation:
@@ -62,32 +55,10 @@ db.pool.config.minimumIdle=2
 ### If enable the instance auto expiration, kind like of health check of instance:
 # nacos.naming.expireInstance=true
 
-### will be removed and replaced by `nacos.naming.clean` properties
 nacos.naming.empty-service.auto-clean=true
 nacos.naming.empty-service.clean.initial-delay-ms=50000
 nacos.naming.empty-service.clean.period-time-ms=30000
 
-### Add in 2.0.0
-### The interval to clean empty service, unit: milliseconds.
-# nacos.naming.clean.empty-service.interval=60000
-
-### The expired time to clean empty service, unit: milliseconds.
-# nacos.naming.clean.empty-service.expired-time=60000
-
-### The interval to clean expired metadata, unit: milliseconds.
-# nacos.naming.clean.expired-metadata.interval=5000
-
-### The expired time to clean metadata, unit: milliseconds.
-# nacos.naming.clean.expired-metadata.expired-time=60000
-
-### The delay time before push task to execute from service changed, unit: milliseconds.
-# nacos.naming.push.pushTaskDelay=500
-
-### The timeout for push task execute, unit: milliseconds.
-# nacos.naming.push.pushTaskTimeout=5000
-
-### The delay time for retrying failed push task, unit: milliseconds.
-# nacos.naming.push.pushTaskRetryDelay=1000
 
 #*************** CMDB Module Related Configurations ***************#
 ### The interval to dump external CMDB in seconds:
@@ -105,7 +76,7 @@ nacos.naming.empty-service.clean.period-time-ms=30000
 
 #*************** Metrics Related Configurations ***************#
 ### Metrics for prometheus
-#management.endpoints.web.exposure.include=*
+management.endpoints.web.exposure.include=*
 
 ### Metrics for elastic search
 management.metrics.export.elastic.enabled=false
@@ -119,7 +90,6 @@ management.metrics.export.influx.enabled=false
 #management.metrics.export.influx.consistency=one
 #management.metrics.export.influx.compressed=true
 
-
 #*************** Access Log Related Configurations ***************#
 ### If turn on the access log:
 server.tomcat.accesslog.enabled=true
@@ -129,8 +99,11 @@ server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b %D %{User-Agent}i %{Reque
 
 ### The directory of access log:
 server.tomcat.basedir=
-
-
+#spring.datasource.platform=mysql
+#db.num=1
+#db.url.0=jdbc:mysql://10.101.167.27:3306/acm?characterEncoding=utf8&connectTimeout=1000&socketTimeout=10000&autoReconnect=true
+#db.user=root
+#db.password=root
 #*************** Access Control Related Configurations ***************#
 ### If enable spring security, this option is deprecated in 1.2.0:
 #spring.security.enabled=false
@@ -141,13 +114,13 @@ nacos.security.ignore.urls=/,/error,/**/*.css,/**/*.js,/**/*.html,/**/*.map,/**/
 ### The auth system to use, currently only 'nacos' and 'ldap' is supported:
 nacos.core.auth.system.type=nacos
 
+### If turn on auth system:
+nacos.core.auth.enabled=false
+
 ### worked when nacos.core.auth.system.type=ldap，{0} is Placeholder,replace login username
 # nacos.core.auth.ldap.url=ldap://localhost:389
 # nacos.core.auth.ldap.userdn=cn={0},ou=user,dc=company,dc=com
 
-### If turn on auth system:
-nacos.core.auth.enabled=false
-
 ### The token expiration in seconds:
 nacos.core.auth.default.token.expire.seconds=18000
 
@@ -158,18 +131,17 @@ nacos.core.auth.default.token.secret.key=SecretKey012345678901234567890123456789
 nacos.core.auth.caching.enabled=true
 
 ### Since 1.4.1, Turn on/off white auth for user-agent: nacos-server, only for upgrade from old version.
-nacos.core.auth.enable.userAgentAuthWhite=true
+nacos.core.auth.enable.userAgentAuthWhite=false
 
 ### Since 1.4.1, worked when nacos.core.auth.enabled=true and nacos.core.auth.enable.userAgentAuthWhite=false.
 ### The two properties is the white list for auth and used by identity the request from other server.
-nacos.core.auth.server.identity.key=
-nacos.core.auth.server.identity.value=
+nacos.core.auth.server.identity.key=serverIdentity
+nacos.core.auth.server.identity.value=security
 
 #*************** Istio Related Configurations ***************#
 ### If turn on the MCP server:
 nacos.istio.mcp.server.enabled=false
 
-
 #*************** Core Related Configurations ***************#
 
 ### set the WorkerID manually
@@ -209,3 +181,23 @@ nacos.istio.mcp.server.enabled=false
 # nacos.core.protocol.raft.data.read_index_type=ReadOnlySafe
 ### rpc request timeout, default 5 seconds
 # nacos.core.protocol.raft.data.rpc_request_timeout_ms=5000
+
+#*************** Distro Related Configurations ***************#
+
+### Distro data sync delay time, when sync task delayed, task will be merged for same data key. Default 1 second.
+# nacos.core.protocol.distro.data.sync.delayMs=1000
+
+### Distro data sync timeout for one sync data, default 3 seconds.
+# nacos.core.protocol.distro.data.sync.timeoutMs=3000
+
+### Distro data sync retry delay time when sync data failed or timeout, same behavior with delayMs, default 3 seconds.
+# nacos.core.protocol.distro.data.sync.retryDelayMs=3000
+
+### Distro data verify interval time, verify synced data whether expired for a interval. Default 5 seconds.
+# nacos.core.protocol.distro.data.verify.intervalMs=5000
+
+### Distro data verify timeout for one verify, default 3 seconds.
+# nacos.core.protocol.distro.data.verify.timeoutMs=3000
+
+### Distro data load retry delay when load snapshot data failed, default 30 seconds.
+# nacos.core.protocol.distro.data.load.retryDelayMs=30000