Skip to content

Commit

Permalink
Update beam to 2.24 (#3325)
Browse files Browse the repository at this point in the history
  • Loading branch information
@regadas
regadas committed Sep 18, 2020
1 parent 6f9c358 commit f49ebf4
Showing 1 changed file with 34 additions and 57 deletions.
91 changes: 34 additions & 57 deletions build.sbt
View file
Expand Up @@ -34,10 +34,10 @@ val autoServiceVersion = "1.0-rc7"
val autoValueVersion = "1.7.4"
val avroVersion = "1.8.2"
val beamVendorVersion = "0.1"
val beamVersion = "2.23.0"
val bigdataossVersion = "2.1.2"
val bigQueryStorageVersion = "0.125.0-beta"
val bigtableClientVersion = "1.13.0"
val beamVersion = "2.24.0"
val bigdataossVersion = "2.1.3"
val bigQueryStorageVersion = "0.133.0-beta"
val bigtableClientVersion = "1.14.0"
val breezeVersion = "1.1"
val caffeineVersion = "2.8.5"
val caseappVersion = "2.0.4"
Expand All @@ -60,13 +60,13 @@ val gcsVersion = "1.8.0"
val generatedGrpcBetaVersion = "1.9.1"
val generatedDatastoreProtoVersion = "0.85.0"
val generatedGrpcGaVersion = "1.85.1"
val googleApiServicesBigQuery = "v2-rev20191211-1.30.9"
val googleApiServicesDataflow = "v1b3-rev20200305-1.30.9"
val googleApiServicesBigQuery = "v2-rev20200719-1.30.10"
val googleApiServicesDataflow = "v1b3-rev20200713-1.30.10"
val googleAuthVersion = "0.19.0"
val googleClientsVersion = "1.30.9"
val googleClientsVersion = "1.30.10"
val googleCloudSpannerVersion = "1.49.1"
val googleHttpClientsVersion = "1.34.0"
val grpcVersion = "1.27.2"
val grpcVersion = "1.29.0"
val guavaVersion = "28.2-jre"
val hadoopVersion = "2.7.7"
val hamcrestVersion = "2.2"
Expand Down Expand Up @@ -115,30 +115,6 @@ lazy val mimaSettings = Seq(
mimaBinaryIssueFilters ++= Seq()
)

val beamSDKIODependencies = Def.settings(
libraryDependencies ++= Seq(
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion excludeAll (
ExclusionRule("com.google.cloud", "google-cloud-spanner"),
ExclusionRule("com.google.cloud.bigtable", "bigtable-client-core"),
ExclusionRule("com.google.cloud", "google-cloud-core-grpc"),
ExclusionRule("io.netty", "netty-tcnative-boringssl-static"),
ExclusionRule("com.google.cloud.bigdataoss", "gcsio")
),
"io.netty" % "netty-tcnative-boringssl-static" % nettyTcNativeVersion,
"com.google.cloud.bigdataoss" % "gcsio" % bigdataossVersion
)
)

val beamSDKGoogleCloudCoreDependencies = Def.settings(
libraryDependencies ++= Seq(
"org.apache.beam" % "beam-sdks-java-extensions-google-cloud-platform-core" % beamVersion
excludeAll (
ExclusionRule("com.google.cloud.bigdataoss", "gcsio")
),
"com.google.cloud.bigdataoss" % "gcsio" % bigdataossVersion
)
)

def previousVersion(currentVersion: String): Option[String] = {
val Version = """(\d+)\.(\d+)\.(\d+).*""".r
val Version(x, y, z) = currentVersion
Expand Down Expand Up @@ -314,6 +290,7 @@ lazy val assemblySettings = Seq(
case s if s.endsWith("libsnappyjava.jnilib") => MergeStrategy.last
case s if s.endsWith("libsnappyjava.so") => MergeStrategy.last
case s if s.endsWith("snappyjava_snappy.dll") => MergeStrategy.last
case s if s.endsWith("reflection-config.json") => MergeStrategy.rename
case s if s.endsWith(".dtd") => MergeStrategy.rename
case s if s.endsWith(".xsd") => MergeStrategy.rename
case PathList("META-INF", "services", "org.apache.hadoop.fs.FileSystem") =>
Expand Down Expand Up @@ -481,6 +458,8 @@ lazy val `scio-core`: Project = project
"org.apache.avro" % "avro" % avroVersion,
"org.apache.beam" % "beam-runners-core-construction-java" % beamVersion,
"org.apache.beam" % "beam-runners-google-cloud-dataflow-java" % beamVersion % Provided,
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
"org.apache.beam" % "beam-sdks-java-extensions-google-cloud-platform-core" % beamVersion,
"org.apache.beam" % "beam-runners-spark" % beamVersion % Provided exclude (
"com.fasterxml.jackson.module", "jackson-module-scala_2.11"
),
Expand All @@ -500,9 +479,7 @@ lazy val `scio-core`: Project = project
"org.typelevel" %% "algebra" % algebraVersion,
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"com.propensive" %% "magnolia" % magnoliaVersion
),
beamSDKIODependencies,
beamSDKGoogleCloudCoreDependencies
)
)
.dependsOn(
`scio-schemas` % "test->test",
Expand Down Expand Up @@ -544,6 +521,7 @@ lazy val `scio-test`: Project = project
libraryDependencies ++= Seq(
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"org.apache.beam" % "beam-runners-direct-java" % beamVersion,
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
"org.apache.beam" % "beam-runners-google-cloud-dataflow-java" % beamVersion % "test,it",
"org.apache.beam" % "beam-sdks-java-core" % beamVersion % "test",
"org.apache.beam" % "beam-sdks-java-core" % beamVersion % "test" classifier "tests",
Expand Down Expand Up @@ -571,7 +549,6 @@ lazy val `scio-test`: Project = project
"org.scalactic" %% "scalactic" % "3.2.2",
"com.propensive" %% "magnolia" % magnoliaVersion
),
beamSDKIODependencies,
(Test / compileOrder) := CompileOrder.JavaThenScala,
Test / testGrouping := splitTests(
(Test / definedTests).value,
Expand Down Expand Up @@ -615,6 +592,7 @@ lazy val `scio-avro`: Project = project
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"me.lyh" %% "protobuf-generic" % protobufGenericVersion,
"org.apache.beam" % "beam-vendor-guava-26_0-jre" % beamVendorVersion,
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
"org.apache.beam" % "beam-sdks-java-core" % beamVersion,
"com.twitter" %% "chill" % chillVersion,
"com.google.protobuf" % "protobuf-java" % protobufVersion,
Expand All @@ -626,8 +604,7 @@ lazy val `scio-avro`: Project = project
"org.scalacheck" %% "scalacheck" % scalacheckVersion % "test,it",
"com.spotify" %% "magnolify-cats" % magnolifyVersion % "test",
"com.spotify" %% "magnolify-scalacheck" % magnolifyVersion % "test"
),
beamSDKIODependencies
)
)
.dependsOn(
`scio-core` % "compile;it->it"
Expand All @@ -645,14 +622,16 @@ lazy val `scio-bigquery`: Project = project
libraryDependencies ++= Seq(
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"org.apache.beam" % "beam-vendor-guava-26_0-jre" % beamVendorVersion,
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
"org.apache.beam" % "beam-sdks-java-extensions-google-cloud-platform-core" % beamVersion,
"com.twitter" %% "chill" % chillVersion,
"com.google.protobuf" % "protobuf-java" % protobufVersion,
"org.apache.avro" % "avro" % avroVersion,
"com.google.api" % "gax" % gaxVersion,
"com.google.api" % "gax-grpc" % gaxVersion,
"com.google.api-client" % "google-api-client" % googleClientsVersion,
"com.google.apis" % "google-api-services-bigquery" % googleApiServicesBigQuery,
"com.google.api.grpc" % "proto-google-cloud-bigquerystorage-v1beta1" % "0.85.1",
"com.google.api.grpc" % "proto-google-cloud-bigquerystorage-v1beta1" % "0.98.0",
"com.google.http-client" % "google-http-client" % googleHttpClientsVersion,
"com.google.http-client" % "google-http-client-jackson" % "1.29.2",
"com.google.http-client" % "google-http-client-jackson2" % googleHttpClientsVersion,
Expand All @@ -673,9 +652,7 @@ lazy val `scio-bigquery`: Project = project
// DataFlow testing requires junit and hamcrest
"org.hamcrest" % "hamcrest-core" % hamcrestVersion % "test,it",
"org.hamcrest" % "hamcrest-library" % hamcrestVersion % "test,it"
),
beamSDKIODependencies,
beamSDKGoogleCloudCoreDependencies
)
)
.dependsOn(
`scio-core` % "compile;it->it"
Expand All @@ -691,6 +668,7 @@ lazy val `scio-bigtable`: Project = project
libraryDependencies ++= Seq(
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"org.apache.beam" % "beam-sdks-java-core" % beamVersion,
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
"joda-time" % "joda-time" % jodaTimeVersion,
"com.google.protobuf" % "protobuf-java" % protobufVersion,
"com.google.api.grpc" % "proto-google-cloud-bigtable-v2" % generatedGrpcBetaVersion,
Expand All @@ -706,8 +684,7 @@ lazy val `scio-bigtable`: Project = project
"com.google.api.grpc" % "proto-google-cloud-bigtable-admin-v2" % generatedGrpcBetaVersion,
"com.google.guava" % "guava" % guavaVersion,
"org.slf4j" % "slf4j-api" % slf4jVersion
),
beamSDKIODependencies
)
)
.dependsOn(
`scio-core`,
Expand Down Expand Up @@ -820,6 +797,7 @@ lazy val `scio-extra`: Project = project
libraryDependencies ++= Seq(
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"org.apache.beam" % "beam-sdks-java-core" % beamVersion,
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
"org.apache.beam" % "beam-sdks-java-extensions-sorter" % beamVersion,
"com.google.apis" % "google-api-services-bigquery" % googleApiServicesBigQuery,
"org.apache.avro" % "avro" % avroVersion,
Expand All @@ -843,7 +821,6 @@ lazy val `scio-extra`: Project = project
"io.circe" %% "circe-generic" % circeVersion,
"io.circe" %% "circe-parser" % circeVersion
),
beamSDKIODependencies,
Compile / sourceDirectories := (Compile / sourceDirectories).value
.filterNot(_.getPath.endsWith("/src_managed/main")),
Compile / managedSourceDirectories := (Compile / managedSourceDirectories).value
Expand Down Expand Up @@ -926,12 +903,12 @@ lazy val `scio-spanner`: Project = project
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"com.google.cloud" % "google-cloud-core" % "1.92.2",
"org.apache.beam" % "beam-sdks-java-core" % beamVersion,
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
"com.google.cloud" % "google-cloud-spanner" % googleCloudSpannerVersion excludeAll (
ExclusionRule(organization = "io.grpc")
),
"org.scalatest" %% "scalatest" % scalatestVersion % "it"
),
beamSDKIODependencies
)
)
.dependsOn(
`scio-core`,
Expand Down Expand Up @@ -1007,6 +984,8 @@ lazy val `scio-examples`: Project = project
libraryDependencies ++= Seq(
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"org.apache.beam" % "beam-sdks-java-core" % beamVersion,
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
"org.apache.beam" % "beam-sdks-java-extensions-google-cloud-platform-core" % beamVersion,
"org.apache.avro" % "avro" % avroVersion,
"com.google.cloud.datastore" % "datastore-v1-proto-client" % datastoreV1ProtoClientVersion,
"com.google.http-client" % "google-http-client" % googleHttpClientsVersion,
Expand Down Expand Up @@ -1043,8 +1022,6 @@ lazy val `scio-examples`: Project = project
"org.elasticsearch" % "elasticsearch" % elasticsearch7Version,
"com.propensive" %% "magnolia" % magnoliaVersion
),
beamSDKIODependencies,
beamSDKGoogleCloudCoreDependencies,
// exclude problematic sources if we don't have GCP credentials
excludeFilter in unmanagedSources := {
if (BuildCredentials.exists) {
Expand Down Expand Up @@ -1085,6 +1062,8 @@ lazy val `scio-repl`: Project = project
libraryDependencies ++= Seq(
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"org.apache.beam" % "beam-runners-direct-java" % beamVersion,
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
"org.apache.beam" % "beam-sdks-java-extensions-google-cloud-platform-core" % beamVersion,
"org.apache.beam" % "beam-runners-google-cloud-dataflow-java" % beamVersion excludeAll (
ExclusionRule("com.google.cloud.bigdataoss", "gcsio")
),
Expand All @@ -1099,8 +1078,6 @@ lazy val `scio-repl`: Project = project
"org.scala-lang" % "scala-compiler" % scalaVersion.value,
"com.nrinaudo" %% "kantan.csv" % kantanCsvVersion
),
beamSDKIODependencies,
beamSDKGoogleCloudCoreDependencies,
libraryDependencies ++= {
VersionNumber(scalaVersion.value) match {
case v if v.matchesSemVer(SemanticSelector("2.12.x")) =>
Expand Down Expand Up @@ -1152,6 +1129,7 @@ lazy val `scio-smb`: Project = project
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"org.apache.beam" % "beam-sdks-java-core" % beamVersion,
"org.apache.beam" % "beam-sdks-java-core" % beamVersion % "it,test" classifier "tests",
"org.apache.beam" % "beam-sdks-java-io-google-cloud-platform" % beamVersion,
// #3260 work around for sorter memory limit until we patch upstream
// "org.apache.beam" % "beam-sdks-java-extensions-sorter" % beamVersion,
"org.apache.beam" % "beam-sdks-java-extensions-protobuf" % beamVersion,
Expand All @@ -1174,7 +1152,6 @@ lazy val `scio-smb`: Project = project
"org.slf4j" % "slf4j-api" % slf4jVersion,
"com.github.ben-manes.caffeine" % "caffeine" % caffeineVersion % "provided"
),
beamSDKIODependencies,
javacOptions ++= {
(Compile / sourceManaged).value.mkdirs()
Seq("-s", (Compile / sourceManaged).value.getAbsolutePath)
Expand Down Expand Up @@ -1340,18 +1317,18 @@ ThisBuild / dependencyOverrides ++= Seq(
"com.google.api.grpc" % "proto-google-common-protos" % "1.17.0",
"com.google.api" % "gax-grpc" % gaxVersion,
"com.google.api" % "gax" % gaxVersion,
"com.google.apis" % "google-api-services-storage" % "v1-rev20200226-1.30.9",
"com.google.apis" % "google-api-services-storage" % "v1-rev20200611-1.30.10",
"com.google.auth" % "google-auth-library-credentials" % googleAuthVersion,
"com.google.auth" % "google-auth-library-oauth2-http" % googleAuthVersion,
"com.google.auto.value" % "auto-value-annotations" % autoValueVersion,
"com.google.cloud.bigdataoss" % "gcsio" % bigdataossVersion,
"com.google.cloud.bigdataoss" % "util" % bigdataossVersion,
"com.google.cloud" % "google-cloud-core-grpc" % "1.92.2",
"com.google.cloud" % "google-cloud-core-http" % "1.92.2",
"com.google.cloud" % "google-cloud-core" % "1.92.2",
"com.google.cloud" % "google-cloud-core-grpc" % "1.93.7",
"com.google.cloud" % "google-cloud-core-http" % "1.93.7",
"com.google.cloud" % "google-cloud-core" % "1.93.7",
"com.google.cloud" % "google-cloud-storage" % gcsVersion,
"com.google.code.findbugs" % "jsr305" % "3.0.2",
"com.google.code.gson" % "gson" % "2.8.5",
"com.google.code.gson" % "gson" % "2.8.6",
"com.google.errorprone" % "error_prone_annotations" % "2.3.4",
"com.google.guava" % "guava" % guavaVersion,
"com.google.http-client" % "google-http-client-jackson2" % googleHttpClientsVersion,
Expand Down

0 comments on commit f49ebf4

Please sign in to comment.