Skip to content

Commit

Permalink
Migrate from sbt-protobuf to sbt-protoc.
Browse files Browse the repository at this point in the history
Setup project so schemas can be generated in Test configuration
  • Loading branch information
RustedBones committed Aug 1, 2022
1 parent d5d128f commit 8b52171
Show file tree
Hide file tree
Showing 9 changed files with 31 additions and 105 deletions.
56 changes: 15 additions & 41 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -413,14 +413,20 @@ def beamRunnerSettings: Seq[Setting[_]] = Seq(
libraryDependencies ++= beamRunnersEval.value
)

lazy val protobufSettings = Def.settings(
ProtobufConfig / version := protobufVersion,
ProtobufConfig / protobufRunProtoc := (args =>
com.github.os72.protocjar.Protoc.runProtoc("-v3.17.3" +: args.toArray)
),
libraryDependencies += "com.google.protobuf" % "protobuf-java" % (ProtobufConfig / version).value % ProtobufConfig.name
lazy val scopedProtobufSettings = Def.settings(
PB.targets := Seq(
PB.gens.java -> (ThisScope.copy(config = Zero) / sourceManaged).value / "compiled_proto" / configuration.value.name,
PB.gens.plugin("grpc-java") -> (ThisScope.copy(config = Zero) / sourceManaged).value / "compiled_grpc" / configuration.value.name
)
)

lazy val protobufSettings = Def.settings(
libraryDependencies ++= Seq(
"io.grpc" % "protoc-gen-grpc-java" % grpcVersion asProtocPlugin(),
"com.google.protobuf" % "protobuf-java" % protobufVersion % "protobuf"
)
) ++ Seq(Compile, Test).flatMap(c => inConfig(c)(scopedProtobufSettings))

def splitTests(tests: Seq[TestDefinition], filter: Seq[String], forkOptions: ForkOptions) = {
val (filtered, default) = tests.partition(test => filter.contains(test.name))
val policy = Tests.SubProcess(forkOptions)
Expand Down Expand Up @@ -449,7 +455,6 @@ lazy val root: Project = Project("scio", file("."))
`scio-jdbc`,
`scio-parquet`,
`scio-tensorflow`,
`scio-schemas`,
`scio-examples`,
`scio-repl`,
`scio-jmh`,
Expand Down Expand Up @@ -517,13 +522,8 @@ lazy val `scio-core`: Project = project
buildInfoKeys := Seq[BuildInfoKey](scalaVersion, version, "beamVersion" -> beamVersion),
buildInfoPackage := "com.spotify.scio"
)
.dependsOn(
`scio-schemas` % "test->test",
`scio-macros`
)
.configs(
IntegrationTest
)
.dependsOn(`scio-macros`)
.configs(IntegrationTest)
.enablePlugins(BuildInfoPlugin)

lazy val `scio-test`: Project = project
Expand All @@ -532,6 +532,7 @@ lazy val `scio-test`: Project = project
.settings(publishSettings)
.settings(itSettings)
.settings(macroSettings)
.settings(protobufSettings)
.settings(
description := "Scio helpers for ScalaTest",
libraryDependencies ++= Seq(
Expand Down Expand Up @@ -576,7 +577,6 @@ lazy val `scio-test`: Project = project
.configs(IntegrationTest)
.dependsOn(
`scio-core` % "test->test;compile->compile;it->it",
`scio-schemas` % "test;it",
`scio-avro` % "compile->test;it->it"
)

Expand Down Expand Up @@ -679,7 +679,6 @@ lazy val `scio-google-cloud-platform`: Project = project
)
.dependsOn(
`scio-core` % "compile;it->it",
`scio-schemas` % "test",
`scio-avro` % "test",
`scio-test` % "test;it"
)
Expand Down Expand Up @@ -907,7 +906,6 @@ lazy val `scio-parquet`: Project = project
.dependsOn(
`scio-core`,
`scio-avro`,
`scio-schemas` % "test",
`scio-test` % "test->test"
)

Expand Down Expand Up @@ -944,28 +942,6 @@ lazy val `scio-tensorflow`: Project = project
`scio-core`,
`scio-test` % "test->test"
)
.enablePlugins(ProtobufPlugin)

lazy val `scio-schemas`: Project = project
.in(file("scio-schemas"))
.settings(commonSettings)
.settings(protobufSettings)
.settings(
description := "Avro/Proto schemas for testing",
publish / skip := true,
mimaPreviousArtifacts := Set.empty,
libraryDependencies ++= Seq(
"org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion,
"org.apache.avro" % "avro" % avroVersion
),
Compile / sourceDirectories := (Compile / sourceDirectories).value
.filterNot(_.getPath.endsWith("/src_managed/main")),
Compile / managedSourceDirectories := (Compile / managedSourceDirectories).value
.filterNot(_.getPath.endsWith("/src_managed/main")),
Compile / doc / sources := List(), // suppress warnings
compileOrder := CompileOrder.JavaThenScala
)
.enablePlugins(ProtobufPlugin)

lazy val `scio-examples`: Project = project
.in(file("scio-examples"))
Expand Down Expand Up @@ -1042,7 +1018,6 @@ lazy val `scio-examples`: Project = project
.dependsOn(
`scio-core`,
`scio-google-cloud-platform`,
`scio-schemas`,
`scio-jdbc`,
`scio-extra`,
`scio-elasticsearch8`,
Expand Down Expand Up @@ -1223,7 +1198,6 @@ lazy val site: Project = project
`scio-avro`,
`scio-google-cloud-platform`,
`scio-parquet`,
`scio-schemas`,
`scio-smb`,
`scio-test`,
`scio-extra`
Expand Down
35 changes: 16 additions & 19 deletions project/plugins.sbt
Original file line number Diff line number Diff line change
@@ -1,28 +1,25 @@
addDependencyTreePlugin
addSbtPlugin("com.github.sbt" % "sbt-avro" % "3.4.0")
addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.5.2")
addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.10.1")
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0")
addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0")
addSbtPlugin("com.github.sbt" % "sbt-protobuf" % "0.7.1")
addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.3")
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0")
addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3")
addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "1.1.0")
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0")
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6")
addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.10.1")
addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1")
addSbtPlugin("com.github.cb372" % "sbt-explicit-dependencies" % "0.2.16")
addSbtPlugin("com.github.sbt" % "sbt-avro" % "3.4.2")
addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.10")
addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0")
addSbtPlugin("com.lightbend.paradox" % "sbt-paradox" % "0.10.2")
addSbtPlugin("io.github.jonas" % "sbt-paradox-material-theme" % "0.6.0")
addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.5.2")
addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.3.2")
addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.7.0")
addSbtPlugin("com.github.cb372" % "sbt-explicit-dependencies" % "0.2.16")
addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.0")
addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2")
addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "1.1.0")
addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.3")
addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1")
addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.7.0")
addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.10")
addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.4.0")
addSbtPlugin("io.github.jonas" % "sbt-paradox-material-theme" % "0.6.0")
addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.3.2")
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6")
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0")
addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3")

libraryDependencies ++= Seq(
"com.github.os72" % "protoc-jar" % "3.11.4",
"org.apache.avro" % "avro-compiler" % "1.8.2"
)
libraryDependencies += "org.apache.avro" % "avro-compiler" % "1.8.2"
45 changes: 0 additions & 45 deletions scio-schemas/src/main/avro/bytes.avsc

This file was deleted.

File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.

0 comments on commit 8b52171

Please sign in to comment.