/
build.sbt
107 lines (99 loc) · 3.76 KB
/
build.sbt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
organization := "com.spotify.data"
name := "gcs-tools"
version := "0.1.8-SNAPSHOT"
val gcsVersion = "1.6.2-hadoop2"
val hadoopVersion = "2.7.4"
val avroVersion = "1.8.2"
val parquetVersion = "1.9.0"
val protobufVersion = "3.4.0"
val protobufGenericVersion = "0.2.4"
val commonSettings = Project.defaultSettings ++ assemblySettings ++ Seq(
scalaVersion := "2.11.12",
autoScalaLibrary := false
)
lazy val root: Project = Project(
"gcs-tools",
file(".")
).aggregate(
avroTools,
parquetTools,
protoTools
)
lazy val shared: Project = Project(
"shared",
file("shared"),
settings = commonSettings
)
lazy val avroTools: Project = Project(
"avro-tools",
file("avro-tools"),
settings = commonSettings ++ Seq(
mainClass in assembly := Some("org.apache.avro.tool.Main"),
assemblyJarName in assembly := s"avro-tools-$avroVersion.jar",
libraryDependencies ++= Seq(
"org.apache.avro" % "avro-tools" % avroVersion,
"com.google.cloud.bigdataoss" % "gcs-connector" % gcsVersion
)
)
).dependsOn(
shared
)
lazy val parquetTools: Project = Project(
"parquet-tools",
file("parquet-tools"),
settings = commonSettings ++ Seq(
mainClass in assembly := Some("org.apache.parquet.tools.Main"),
assemblyJarName in assembly := s"parquet-tools-$parquetVersion.jar",
libraryDependencies ++= Seq(
"org.apache.parquet" % "parquet-tools" % parquetVersion,
"org.apache.hadoop" % "hadoop-common" % hadoopVersion,
"com.google.cloud.bigdataoss" % "gcs-connector" % gcsVersion
)
)
).dependsOn(
shared
)
lazy val protoTools: Project = Project(
"proto-tools",
file("proto-tools"),
settings = commonSettings ++ Seq(
mainClass in assembly := Some("org.apache.avro.tool.ProtoMain"),
assemblyJarName in assembly := s"proto-tools-$protobufVersion.jar",
assemblyShadeRules in assembly := Seq(
ShadeRule.zap("com.google.protobuf.**").inLibrary("org.apache.avro" % "avro-tools" % avroVersion)
),
libraryDependencies ++= Seq(
"me.lyh" %% "protobuf-generic" % protobufGenericVersion exclude("com.google.guava", "guava"),
"com.google.protobuf" % "protobuf-java" % protobufVersion,
"org.apache.avro" % "avro-tools" % avroVersion,
"com.google.cloud.bigdataoss" % "gcs-connector" % gcsVersion
)
)
).dependsOn(
shared
)
lazy val assemblySettings = Seq(
mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) => {
case s if s.endsWith(".properties") => MergeStrategy.filterDistinctLines
case s if s.endsWith("pom.xml") => MergeStrategy.last
case s if s.endsWith(".class") => MergeStrategy.last
case s if s.endsWith("libjansi.jnilib") => MergeStrategy.last
case s if s.endsWith("jansi.dll") => MergeStrategy.rename
case s if s.endsWith("libjansi.so") => MergeStrategy.rename
case s if s.endsWith("libsnappyjava.jnilib") => MergeStrategy.last
case s if s.endsWith("libsnappyjava.so") => MergeStrategy.last
case s if s.endsWith("snappyjava_snappy.dll") => MergeStrategy.last
case s if s.endsWith(".dtd") => MergeStrategy.rename
case s if s.endsWith(".xsd") => MergeStrategy.rename
case PathList("META-INF", "services", "org.apache.hadoop.fs.FileSystem") => MergeStrategy.filterDistinctLines
case PathList("META-INF", "LICENSE") => MergeStrategy.discard
case PathList("META-INF", "MANIFEST.MF") => MergeStrategy.discard
case PathList("META-INF", "DUMMY.SF") => MergeStrategy.discard
case PathList("META-INF", "DUMMY.RSA") => MergeStrategy.discard
case PathList("META-INF", "DUMMY.DSA") => MergeStrategy.discard
case PathList("META-INF", "MSFTSIG.RSA") => MergeStrategy.discard
case PathList("META-INF", "MSFTSIG.SF") => MergeStrategy.discard
case PathList("META-INF", "NOTICE") => MergeStrategy.rename
case _ => MergeStrategy.last
}}
)