jarファイルを１つにまとめた。コンフィグ読み込み部分を修正した。

ken57 · Sep 5, 2016 · 691e287 · 691e287
1 parent fa2a72e
commit 691e287
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 8 deletions.
diff --git a/dga-graphx/build.gradle b/dga-graphx/build.gradle
@@ -25,6 +25,17 @@ sourceSets {
     }
 }
 
+buildscript {
+    repositories {
+        jcenter()
+    }
+    dependencies {
+        classpath 'eu.appsatori:gradle-fatjar-plugin:0.3'
+    }
+}
+
+apply plugin: 'eu.appsatori.fatjar'
+
 dependencies {
     compile project(':dga-core')
     compile group: 'org.apache.spark', name: 'spark-core_2.10', version: '1.3.0'
@@ -88,4 +99,9 @@ task dist(dependsOn: 'distJars', type: Copy) {
     include "dga-mr1-graphx"
     include "dga-yarn-graphx"
     into "${buildDir}/dist"
-}
+}
+
+fatJar {
+  exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA' 
+}
+
diff --git a/dga-graphx/src/main/scala/com/soteradefense/dga/graphx/DGARunner.scala b/dga-graphx/src/main/scala/com/soteradefense/dga/graphx/DGARunner.scala
@@ -51,7 +51,7 @@ object DGARunner {
   val NeighboringCommunities = "neighboringCommunities"
 
   val MinProgressConfiguration = "minProgress"
-  val MinProgressDefaultConfiguration = "2000"
+  val MinProgressDefaultConfiguration = "0"
   val ProgressCounterConfiguration = "progressCounter"
   val ProgressCounterDefaultConfiguration = "1"
   val DeltaConvergenceConfiguration = "delta"
@@ -81,7 +81,7 @@ object DGARunner {
     val sparkContext = new SparkContext(sparkConf)
     val parallelism = Integer.parseInt(commandLineConfig.customArguments.getOrElse(ParallelismConfiguration, applicationConfig.getString("parallelism")))
     var inputFormat: EdgeInputFormat = null
-    val hdfsUrl = applicationConfig.getString("hdfs.url")
+    val hdfsUrl = commandLineConfig.customArguments.getOrElse("hdfs.url", applicationConfig.getString("hdfs.url"))
     val inputPath = hdfsUrl + commandLineConfig.inputPath
     var outputPath = hdfsUrl + commandLineConfig.outputPath
     outputPath = if (outputPath.endsWith("/")) outputPath else outputPath + "/"
@@ -133,11 +133,9 @@ object DGARunner {
   private def buildSparkConf(commandLineConfig: Config, applicationConfig: com.typesafe.config.Config): SparkConf = {
     val sparkConf = new SparkConf()
       .setAppName(commandLineConfig.sparkAppName)
-      .setJars(commandLineConfig.sparkJars.split(DefaultJarSplitDelimiter))
-    if (applicationConfig.hasPath("spark.master.url"))
-      sparkConf.setMaster(applicationConfig.getString("spark.master.url"))
-    if (applicationConfig.hasPath("spark.home"))
-      sparkConf.setSparkHome(applicationConfig.getString("spark.home"))
+      .setJars(Seq("dga-graphx-0.1.jar"))
+    sparkConf.setMaster(commandLineConfig.customArguments.getOrElse("spark.master.url", applicationConfig.getString("spark.master.url")))
+    sparkConf.setSparkHome(commandLineConfig.customArguments.getOrElse("spark.home", applicationConfig.getString("spark.home")))
     sparkConf.setAll(commandLineConfig.customArguments)
     if (commandLineConfig.useKryoSerializer) {
       sparkConf.set("spark.serializer", classOf[KryoSerializer].getCanonicalName)