extract common code for spark3.0

vesoft-inc · Dec 21, 2021 · 858d134 · 858d134
1 parent e24879e
commit 858d134
Show file tree

Hide file tree

Showing 22 changed files with 35 additions and 1,885 deletions.
diff --git a/nebula-exchange_spark_3.0/pom.xml b/nebula-exchange_spark_3.0/pom.xml
@@ -16,7 +16,7 @@
         <maven.compiler.target>1.8</maven.compiler.target>
         <spark.version>3.0.0</spark.version>
         <version.scala.binary>2.12</version.scala.binary>
-        <scala.version>2.12.13</scala.version>
+        <scala.version>2.12.10</scala.version>
         <spark-csv.version>1.5.0</spark-csv.version>
         <scalatest.version>3.2.0</scalatest.version>
         <scala-logging.version>3.9.2</scala-logging.version>

diff --git a/nebula-exchange_spark_3.0/src/main/scala/com/vesoft/nebula/exchange/CheckPointHandler.scala b/nebula-exchange_spark_3.0/src/main/scala/com/vesoft/nebula/exchange/CheckPointHandler.scala
@@ -5,9 +5,9 @@
 
 package com.vesoft.nebula.exchange
 
-import com.vesoft.nebula.exchange.config.{SourceCategory}
-import com.vesoft.nebula.exchange.utils.HDFSUtils
-import com.vesoft.nebula.exchange.config.SchemaConfigEntry
+import com.vesoft.nebula.common.config.SourceCategory
+import com.vesoft.nebula.common.config.SchemaConfigEntry
+import com.vesoft.nebula.common.utils.HDFSUtils
 import org.apache.spark.TaskContext
 
 /**

diff --git a/nebula-exchange_spark_3.0/src/main/scala/com/vesoft/nebula/exchange/Exchange.scala b/nebula-exchange_spark_3.0/src/main/scala/com/vesoft/nebula/exchange/Exchange.scala
@@ -7,7 +7,9 @@ package com.vesoft.nebula.exchange
 
 import org.apache.spark.sql.{DataFrame, SparkSession}
 import java.io.File
-import com.vesoft.nebula.exchange.config.{
+
+import com.vesoft.nebula.common.Argument
+import com.vesoft.nebula.common.config.{
   ClickHouseConfigEntry,
   Configs,
   DataSourceConfigEntry,
@@ -23,7 +25,7 @@ import com.vesoft.nebula.exchange.config.{
   SinkCategory,
   SourceCategory
 }
-import com.vesoft.nebula.exchange.processor.{EdgeProcessor, VerticesProcessor}
+
 import com.vesoft.nebula.exchange.reader.{
   CSVReader,
   ClickhouseReader,
@@ -39,16 +41,12 @@ import com.vesoft.nebula.exchange.reader.{
   ParquetReader,
   PulsarReader
 }
+import com.vesoft.nebula.exchange.processor.EdgeProcessor
 import com.vesoft.nebula.exchange.processor.ReloadProcessor
+import com.vesoft.nebula.exchange.processor.VerticesProcessor
 import org.apache.log4j.Logger
 import org.apache.spark.SparkConf
 
-final case class Argument(config: String = "application.conf",
-                          hive: Boolean = false,
-                          directly: Boolean = false,
-                          dry: Boolean = false,
-                          reload: String = "")
-
 final case class TooManyErrorsException(private val message: String) extends Exception(message)
 
 /**

diff --git a/nebula-exchange_spark_3.0/src/main/scala/com/vesoft/nebula/exchange/GraphProvider.scala b/nebula-exchange_spark_3.0/src/main/scala/com/vesoft/nebula/exchange/GraphProvider.scala
diff --git a/nebula-exchange_spark_3.0/src/main/scala/com/vesoft/nebula/exchange/MetaProvider.scala b/nebula-exchange_spark_3.0/src/main/scala/com/vesoft/nebula/exchange/MetaProvider.scala