forked from allwefantasy/spark-deep-learning-toy
-
Notifications
You must be signed in to change notification settings - Fork 0
/
build.sbt
94 lines (70 loc) · 2.94 KB
/
build.sbt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
// Your sbt build file. Guides on how to write one can be found at
// http://www.scala-sbt.org/0.13/docs/index.html
import ReleaseTransformations._
val sparkVer = sys.props.getOrElse("spark.version", "2.3.1")
val sparkBranch = sparkVer.substring(0, 3)
val defaultScalaVer = sparkBranch match {
case "2.3" => "2.11.8"
case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.")
}
val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer)
val scalaMajorVersion = scalaVer.substring(0, scalaVer.indexOf(".", scalaVer.indexOf(".") + 1))
sparkVersion := sparkVer
scalaVersion := scalaVer
spName := "databricks/spark-deep-learning"
// Don't forget to set the version
version := (version in ThisBuild).value + s"-spark$sparkBranch"
// All Spark Packages need a license
licenses := Seq("Apache-2.0" -> url("http://opensource.org/licenses/Apache-2.0"))
spAppendScalaVersion := true
// Add Spark components this package depends on, e.g, "mllib", ....
sparkComponents ++= Seq("mllib-local", "mllib", "sql")
// uncomment and change the value below to change the directory where your zip artifact will be created
// spDistDirectory := target.value
// add any Spark Package dependencies using spDependencies.
// e.g. spDependencies += "databricks/spark-avro:0.1"
spDependencies += s"databricks/tensorframes:0.5.0-s_${scalaMajorVersion}"
libraryDependencies ++= Seq(
// Update to scala-logging 3.9.0 after we update TensorFrames.
"com.typesafe.scala-logging" %% "scala-logging-api" % "2.1.2",
"com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2",
// Matching scalatest versions from TensorFrames
"org.scalactic" %% "scalactic" % "3.0.0" % "test",
"org.scalatest" %% "scalatest" % "3.0.0" % "test"
)
libraryDependencies += "graphframes" % "graphframes" % "0.6.0-spark2.3-s_2.11"
libraryDependencies += "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0"
assemblyMergeStrategy in assembly := {
case "requirements.txt" => MergeStrategy.concat
case x =>
val oldStrategy = (assemblyMergeStrategy in assembly).value
oldStrategy(x)
}
parallelExecution := false
scalacOptions in(Compile, doc) ++= Seq(
"-groups",
"-implicits",
"-skip-packages", Seq("org.apache.spark").mkString(":"))
scalacOptions in(Test, doc) ++= Seq("-groups", "-implicits")
// This fixes a class loader problem with scala.Tuple2 class, scala-2.11, Spark 2.x
fork in Test := true
// This and the next line fix a problem with forked run: https://github.com/scalatest/scalatest/issues/770
javaOptions in Test ++= Seq(
"-Xmx2048m",
"-XX:ReservedCodeCacheSize=384m",
"-XX:MaxPermSize=384m",
"-Djava.awt.headless=true"
)
concurrentRestrictions in Global := Seq(
Tags.limitAll(1))
autoAPIMappings := true
coverageHighlighting := false
// We only use sbt-release to update version numbers for now.
releaseProcess := Seq[ReleaseStep](
inquireVersions,
setReleaseVersion,
commitReleaseVersion,
tagRelease,
setNextVersion,
commitNextVersion
)