Lazy load HDFS instance

itinycheng · Jun 14, 2022 · d6a33e7 · d6a33e7
1 parent bc42eb0
commit d6a33e7
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 4 deletions.
diff --git a/flink-platform-web/src/main/java/com/flink/platform/web/config/HadoopConfig.java b/flink-platform-web/src/main/java/com/flink/platform/web/config/HadoopConfig.java
@@ -6,6 +6,7 @@
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Lazy;
 
 import java.io.File;
 import java.nio.file.Path;
@@ -29,6 +30,7 @@ public class HadoopConfig {
 
     private Map<String, String> properties;
 
+    @Lazy
     @Bean("hdfsFileSystem")
     public FileSystem createHdfsFileSystem() throws Exception {
         System.setProperty("HADOOP_USER_NAME", username);
@@ -62,6 +64,7 @@ public String createDataDir() {
         return path.toString();
     }
 
+    @Lazy
     @Bean("projectHdfsPath")
     public String createHdfsFilePath(FileSystem hdfsFileSystem) throws Exception {
         org.apache.hadoop.fs.Path path = new org.apache.hadoop.fs.Path(hdfsFilePath);

diff --git a/flink-platform-web/src/main/java/com/flink/platform/web/runner/FlowExecuteThread.java b/flink-platform-web/src/main/java/com/flink/platform/web/runner/FlowExecuteThread.java
@@ -49,7 +49,7 @@ public FlowExecuteThread(JobFlowRun jobFlowRun, WorkerConfig workerConfig) {
         this.workerConfig = workerConfig;
         this.jobExecService =
                 ThreadUtil.newFixedThreadExecutor(
-                        String.format("FlowExecThread-runId_%d", jobFlowRun.getId()),
+                        String.format("FlowExecThread-flowRunId_%d", jobFlowRun.getId()),
                         workerConfig.getPerFlowExecThreads());
     }
 

diff --git a/flink-platform-web/src/main/java/com/flink/platform/web/service/HdfsService.java b/flink-platform-web/src/main/java/com/flink/platform/web/service/HdfsService.java
@@ -6,10 +6,10 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 
-import javax.annotation.Resource;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -25,7 +25,7 @@
 @Service
 public class HdfsService {
 
-    @Resource private FileSystem fileSystem;
+    @Lazy @Autowired private FileSystem fileSystem;
 
     public void copyFileToLocalIfChanged(Path hdfsFile, Path localFile) throws IOException {
         boolean isCopy = true;