HDFS-16737. Modify code based on comments

apache · Aug 29, 2022 · 845f333 · 845f333
1 parent 4987d2a
commit 845f333
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 3 deletions.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -151,6 +151,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final long    DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0;
   public static final String  DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = "dfs.datanode.fsdatasetcache.max.threads.per.volume";
   public static final int     DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
+  @Deprecated
+  public static final String  DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY =
+      "dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume";
+  @Deprecated
+  public static final int     DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
   public static final String  DFS_DATANODE_FSDATASETASYNCDISK_THREADS_PER_VOLUME_KEY =
       "dfs.datanode.fsdatasetasyncdisk.threads.per.volume";
   public static final int     DFS_DATANODE_FSDATASETASYNCDISK_THREADS_PER_VOLUME_DEFAULT = 1;

diff --git a/...java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/...java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -88,10 +88,10 @@ class FsDatasetAsyncDiskService {
   FsDatasetAsyncDiskService(DataNode datanode, FsDatasetImpl fsdatasetImpl) {
     this.datanode = datanode;
     this.fsdatasetImpl = fsdatasetImpl;
-    threadsPerVolume = datanode.getConf().getInt(
+    this.threadsPerVolume = datanode.getConf().getInt(
       DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_THREADS_PER_VOLUME_KEY,
           DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_THREADS_PER_VOLUME_DEFAULT);
-    Preconditions.checkArgument(threadsPerVolume > 0,
+    Preconditions.checkArgument(this.threadsPerVolume > 0,
         DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_THREADS_PER_VOLUME_KEY +
           " must be a positive integer.");
   }
@@ -114,7 +114,7 @@ public Thread newThread(Runnable r) {
     };
 
     ThreadPoolExecutor executor = new ThreadPoolExecutor(
-        threadsPerVolume, threadsPerVolume,
+        this.threadsPerVolume, this.threadsPerVolume,
         THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
         new LinkedBlockingQueue<Runnable>(), threadFactory);
 

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3002,6 +3002,16 @@
   </description>
 </property>
 
+<property>
+  <name>dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume</name>
+  <value>4</value>
+  <description>
+    The maximum number of threads per volume used to process async disk
+    operations on the datanode. These threads consume I/O and CPU at the
+    same time. This will affect normal data node operations.
+  </description>
+</property>
+
 <property>
   <name>dfs.datanode.fsdatasetasyncdisk.threads.per.volume</name>
   <value>1</value>