[feture-2167] File type variable configuration in FileSource.scala (#…

…2168) * [feture-2167] File type variable configuration in FileSource.scala * [feture-2167] File type variable configuration in FileSource.scala
apache · May 24, 2022 · 8299f98 · 8299f98
1 parent 3aa1145
commit 8299f98
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 2 deletions.
diff --git a/...mons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala b/...mons/linkis-storage/src/main/scala/org/apache/linkis/storage/conf/LinkisStorageConf.scala
@@ -17,10 +17,13 @@
 
 package org.apache.linkis.storage.conf
 
+import org.apache.commons.lang.StringUtils
 import org.apache.linkis.common.conf.{ByteType, CommonVars}
 import org.apache.linkis.common.utils.ByteTimeUtils
 
 object LinkisStorageConf {
+  private val CONF_LOCK = new Object()
+
   val HDFS_FILE_SYSTEM_REST_ERRS: String =
     CommonVars.apply(
       "wds.linkis.hdfs.rest.errs",
@@ -31,4 +34,24 @@ object LinkisStorageConf {
 
   val ROW_BYTE_MAX_LEN = ByteTimeUtils.byteStringAsBytes(ROW_BYTE_MAX_LEN_STR)
 
+  val FILE_TYPE = CommonVars("wds.linkis.storage.file.type", "dolphin,sql,scala,py,hql,python,out,log,text,sh,jdbc,ngql,psql,fql").getValue
+
+  private var fileTypeArr: Array[String] = null
+
+  private def fileTypeArrParser(fileType: String): Array[String] = {
+    if (StringUtils.isBlank(fileType)) Array()
+    else fileType.split(",")
+  }
+
+  def getFileTypeArr: Array[String] = {
+    if (fileTypeArr == null) {
+      CONF_LOCK.synchronized {
+        if (fileTypeArr == null) {
+          fileTypeArr = fileTypeArrParser(FILE_TYPE)
+        }
+      }
+    }
+    fileTypeArr
+  }
+
 }
diff --git a/...s-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala b/...s-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala
@@ -19,13 +19,13 @@ package org.apache.linkis.storage.source
 
 import java.io.{Closeable, InputStream}
 import java.util
-
 import org.apache.linkis.common.io._
 import org.apache.linkis.storage.exception.StorageErrorException
 import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
 import org.apache.linkis.storage.script.ScriptFsReader
 import org.apache.linkis.storage.utils.StorageConfiguration
 import org.apache.commons.math3.util.Pair
+import org.apache.linkis.storage.conf.LinkisStorageConf
 
 
 trait FileSource extends Closeable {
@@ -55,7 +55,7 @@ trait FileSource extends Closeable {
 
 object FileSource {
 
-  private val fileType = Array("dolphin", "sql", "scala", "py", "hql", "python", "out", "log", "text", "sh", "jdbc", "ngql", "psql", "fql")
+  private val fileType = LinkisStorageConf.getFileTypeArr
   private val suffixPredicate = (path: String, suffix: String) => path.endsWith(s".$suffix")
 
   def isResultSet(path: String): Boolean = {