Switch to Spark 0.8.0-incubating

Author: Cecca

README.md

@@ -15,5 +15,6 @@ Branches
 Changelog
 ---------
 
+ - v0.2.0: Switch to Spark 0.8.0-incubating
  - v0.1.0: Implement ball decomposition (deterministic and randomized)
            and the distributed version of HyperANF

build.sbt

@@ -2,7 +2,7 @@ import AssemblyKeys._
 
 name := "spark-graph"
 
-version := "0.1.0"
+version := "0.2.0"
 
 scalaVersion := "2.9.3"
 
@@ -12,9 +12,12 @@ scalacOptions += "-optimise"
 // multiple SparkContexts
 parallelExecution in Test := false
 
-libraryDependencies += "org.apache.hadoop" % "hadoop-client" % "1.2.1"
+// The transitive dependency on "asm" is excluded since hadoop depends
+// on asm-3.1 and everything else on asm-4.0 and they are incompatible.
+// This exclusion fixes the problem.
+libraryDependencies += "org.apache.hadoop" % "hadoop-client" % "1.2.1" exclude("asm","asm")
 
-libraryDependencies += "org.spark-project" % "spark-core_2.9.3" % "0.7.3" excludeAll(
+libraryDependencies += "org.apache.spark" % "spark-core_2.9.3" % "0.8.0-incubating" excludeAll(
     ExclusionRule("ch.qos.logback"),
     ExclusionRule("org.apache.hadoop")
     )
@@ -30,6 +33,8 @@ resolvers ++= Seq(
   "Akka Repository" at "http://repo.akka.io/releases/",
   "Spray Repository" at "http://repo.spray.cc/")
 
+// dependency graph
+net.virtualvoid.sbt.graph.Plugin.graphSettings
 
 // sbt-assembly configuration
 

project/plugins.sbt

@@ -1,2 +1,4 @@
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.9.2")
 
+addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.7.4")
+

src/main/scala/it/unipd/dei/graph/MatToAdjConverter.scala

@@ -17,8 +17,8 @@
 
 package it.unipd.dei.graph
 
-import spark.RDD
-import spark.SparkContext._
+import org.apache.spark.rdd.RDD
+import org.apache.spark.SparkContext._
 
 /**
  * Converts a dataset in sparse matrix form to a dataset in adjacency list form

src/main/scala/it/unipd/dei/graph/Tool.scala

@@ -22,7 +22,7 @@ import it.unipd.dei.graph.decompositions.BallDecomposition._
 import it.unipd.dei.graph.decompositions.FloodBallDecomposition._
 import it.unipd.dei.graph.diameter.hyperAnf.HyperAnf._
 import it.unipd.dei.graph.serialization.KryoSerialization
-import spark.SparkContext
+import org.apache.spark.SparkContext
 import org.slf4j.LoggerFactory
 import it.unipd.dei.graph.decompositions.RandomizedBallDecomposition._
 import it.unipd.dei.graph.decompositions.SimpleRandomizedBallDecomposition._

src/main/scala/it/unipd/dei/graph/decompositions/ArcRelabeler.scala

@@ -17,8 +17,8 @@
 
 package it.unipd.dei.graph.decompositions
 
-import spark.RDD
-import spark.SparkContext._
+import org.apache.spark.rdd.RDD
+import org.apache.spark.SparkContext._
 import it.unipd.dei.graph._
 
 /**

src/main/scala/it/unipd/dei/graph/decompositions/BallComputer.scala

@@ -18,8 +18,8 @@
 package it.unipd.dei.graph.decompositions
 
 import it.unipd.dei.graph._
-import spark.RDD
-import spark.SparkContext._
+import org.apache.spark.rdd.RDD
+import org.apache.spark.SparkContext._
 
 /**
  * Trait for classes that can compute balls

src/main/scala/it/unipd/dei/graph/decompositions/BallDecomposition.scala

@@ -17,8 +17,8 @@
 
 package it.unipd.dei.graph.decompositions
 
-import spark.SparkContext._
-import spark.RDD
+import org.apache.spark.SparkContext._
+import org.apache.spark.rdd.RDD
 import it.unipd.dei.graph._
 import org.slf4j.LoggerFactory
 

src/main/scala/it/unipd/dei/graph/decompositions/FloodBallDecomposition.scala

@@ -18,12 +18,12 @@
 package it.unipd.dei.graph.decompositions
 
 import it.unipd.dei.graph._
-import spark.SparkContext._
-import spark.RDD
+import org.apache.spark.SparkContext._
+import org.apache.spark.rdd.RDD
 import scala.util.Random
 import org.slf4j.LoggerFactory
 import scala.Left
-import spark.broadcast.Broadcast
+import org.apache.spark.broadcast.Broadcast
 
 object FloodBallDecomposition extends ArcRelabeler with Timed {
 

src/main/scala/it/unipd/dei/graph/decompositions/RandomizedBallDecomposition.scala

@@ -19,9 +19,9 @@ package it.unipd.dei.graph.decompositions
 
 import it.unipd.dei.graph._
 import it.unipd.dei.graph.decompositions._
-import spark.SparkContext._
-import spark.RDD
-import spark.broadcast.Broadcast
+import org.apache.spark.SparkContext._
+import org.apache.spark.rdd.RDD
+import org.apache.spark.broadcast.Broadcast
 import scala.util.Random
 import org.slf4j.LoggerFactory
 

src/main/scala/it/unipd/dei/graph/decompositions/SimpleRandomizedBallDecomposition.scala

@@ -19,10 +19,10 @@ package it.unipd.dei.graph.decompositions
 
 import org.slf4j.LoggerFactory
 import it.unipd.dei.graph._
-import spark.RDD
-import spark.SparkContext._
+import org.apache.spark.rdd.RDD
+import org.apache.spark.SparkContext._
 import scala.Left
-import spark.broadcast.Broadcast
+import org.apache.spark.broadcast.Broadcast
 import scala.util.Random
 
 object SimpleRandomizedBallDecomposition extends BallComputer

src/main/scala/it/unipd/dei/graph/diameter/hyperAnf/HyperAnf.scala

@@ -17,13 +17,14 @@
 
 package it.unipd.dei.graph.diameter.hyperAnf
 
-import spark.{Accumulator, RDD, SparkContext}
-import spark.SparkContext._
+import org.apache.spark.{Accumulator, SparkContext}
+import org.apache.spark.SparkContext._
 import it.unipd.dei.graph.{Timed, TextInputConverter, NodeId, Neighbourhood}
 import scala.collection.mutable
 import it.unipd.dei.graph.diameter.{Confidence,EffectiveDiameter}
 import org.slf4j.LoggerFactory
 import java.io.File
+import org.apache.spark.rdd.RDD
 
 /**
  * Implementation of HyperANF with spark

src/main/scala/it/unipd/dei/graph/serialization/KryoSerialization.scala

@@ -20,19 +20,20 @@ package it.unipd.dei.graph.serialization
 import com.esotericsoftware.kryo.Kryo
 import it.unipd.dei.graph.diameter.hyperAnf.HyperLogLogCounter
 import org.slf4j.LoggerFactory
+import org.apache.spark.serializer.KryoRegistrator
 
 /**
  * Trait that enables kryo serialization and registers some classes
  */
 trait KryoSerialization {
 
-  System.setProperty("spark.serializer", "spark.KryoSerializer")
+  System.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
   System.setProperty("spark.kryo.registrator",
     "it.unipd.dei.graph.serialization.GraphKryoRegistrator")
 
 }
 
-class GraphKryoRegistrator extends spark.KryoRegistrator {
+class GraphKryoRegistrator extends KryoRegistrator {
 
   private val log = LoggerFactory.getLogger("KryoRegistrator")
 

src/test/scala/it/unipd/dei/graph/LocalSparkContext.scala

@@ -18,7 +18,7 @@
 package it.unipd.dei.graph
 
 import org.scalatest.{BeforeAndAfterEach, Suite}
-import spark.SparkContext
+import org.apache.spark.SparkContext
 
 /**
  * Provides a SparkContext to each test. The SparkContext is initialized and

src/test/scala/it/unipd/dei/graph/decompositions/BigGraphBallDecompositionSpec.scala

@@ -19,7 +19,7 @@ package it.unipd.dei.graph.decompositions
 
 import org.scalatest._
 import BallDecomposition._
-import spark.RDD
+import org.apache.spark.rdd.RDD
 import scala.collection.mutable
 import it.unipd.dei.graph._
 import it.unipd.dei.graph.LocalSparkContext

src/test/scala/it/unipd/dei/graph/diameter/hyperAnf/HyperAnfSparkSpec.scala

@@ -20,7 +20,7 @@ package it.unipd.dei.graph.diameter.hyperAnf
 import org.scalatest.FlatSpec
 import it.unipd.dei.graph.LocalSparkContext
 import HyperAnf._
-import spark.SparkContext
+import org.apache.spark.SparkContext
 
 class HyperAnfSparkSpec extends FlatSpec with LocalSparkContext {