SBT - Adds AvroWithSchema Support (#233)

* Adds AvroWithSchema support for scala source generation

* Adds scripted tests verifying AvroWithSchema feature

* Updates IDL docs section with the new sbt setting

* Updates/adds unit tests

docs/src/main/tut/idl-generation.md

@@ -225,6 +225,7 @@ sourceGenerators in Compile += (srcGenFromJars in Compile).taskValue)
 Just like `idlGen`, `srcGen` and `srcGenFromJars` has some configurable settings:
 
 * **`idlType`**: the type of IDL to generate from, currently only `avro`.
+* **`srcGenSerializationType`**: the serialization type when generating Scala sources from the IDL definitions. `Protobuf`, `Avro` or `AvroWithSchema` are the current supported serialization types. By default, the serialization type is `Avro`.
 * **`srcJarNames`**: the list of jar names containing the IDL definitions that will be used at compilation time by `srcGenFromJars` to generate the Scala Sources. By default, this sequence is empty.
 * **`srcGenSourceDir`**: the IDL source base directory, where your IDL files are placed. By default: `Compile / resourceDirectory`, typically `src/main/resources/`.
 * **`srcGenTargetDir`**: the Scala target base directory, where the `srcGen` task will write the Scala files in subdirectories/packages based on the namespaces of the IDL files. By default: `Compile / sourceManaged`, typically `target/scala-2.12/src_managed/main/`.

modules/idlgen/core/src/main/scala/Generator.scala

@@ -22,14 +22,20 @@ trait Generator {
 
   def idlType: String
 
-  def generateFrom(files: Set[File], options: String*): Seq[(File, String, Seq[String])] =
+  def generateFrom(
+      files: Set[File],
+      serializationType: String,
+      options: String*): Seq[(File, String, Seq[String])] =
     inputFiles(files).flatMap(inputFile =>
-      generateFrom(inputFile, options: _*).map {
+      generateFrom(inputFile, serializationType, options: _*).map {
         case (outputPath, output) =>
           (inputFile, outputPath, output)
     })
 
   protected def inputFiles(files: Set[File]): Seq[File]
 
-  protected def generateFrom(inputFile: File, options: String*): Option[(String, Seq[String])]
+  protected def generateFrom(
+      inputFile: File,
+      serializationType: String,
+      options: String*): Option[(String, Seq[String])]
 }

modules/idlgen/core/src/main/scala/GeneratorApplication.scala

@@ -31,28 +31,36 @@ abstract class GeneratorApplication[T <: Generator](generators: T*) {
 
   def generateFrom(args: Array[String]): Seq[File] = {
     validate(
-      args.length >= 3,
-      s"Usage: ${getClass.getName.dropRight(1)} idlType inputPath outputPath [option1 option2 ...]")
-    val idlType   = args(0)
-    val inputPath = new File(args(1))
+      args.length >= 4,
+      s"Usage: ${getClass.getName.dropRight(1)} idlType serializationType inputPath outputPath [option1 option2 ...]")
+
+    val idlType           = args(0)
+    val serializationType = args(1)
+
+    validate(
+      serializationTypes.contains(serializationType),
+      s"Unknown Serialization type '$serializationType'. Valid values: ${serializationTypes.mkString(", ")}")
+
+    val inputPath = new File(args(2))
     validate(
       inputPath.exists,
       s"Input path '$inputPath' doesn't exist"
     )
-    val outputDir = new File(args(2))
-    val options   = args.drop(3)
-    generateFrom(idlType, inputPath.allFiles.toSet, outputDir, options: _*)
+    val outputDir = new File(args(3))
+    val options   = args.drop(4)
+    generateFrom(idlType, serializationType, inputPath.allFiles.toSet, outputDir, options: _*)
   }
 
   def generateFrom(
       idlType: String,
+      serializationType: String,
       inputFiles: Set[File],
       outputDir: File,
       options: String*): Seq[File] = {
     validate(
       idlTypes.contains(idlType),
       s"Unknown IDL type '$idlType'. Valid values: ${idlTypes.mkString(", ")}")
-    generatorsByType(idlType).generateFrom(inputFiles, options: _*).map {
+    generatorsByType(idlType).generateFrom(inputFiles, serializationType, options: _*).map {
       case (inputFile, outputFilePath, output) =>
         val outputFile = new File(outputDir, outputFilePath)
         logger.info(s"$inputFile -> $outputFile")

modules/idlgen/core/src/main/scala/IdlGenerator.scala

@@ -29,7 +29,10 @@ trait IdlGenerator extends Generator {
   def inputFiles(files: Set[File]): Seq[File] =
     files.filter(_.getName.endsWith(ScalaFileExtension)).toSeq
 
-  def generateFrom(inputFile: File, options: String*): Option[(String, Seq[String])] = {
+  def generateFrom(
+      inputFile: File,
+      serializationType: String,
+      options: String*): Option[(String, Seq[String])] = {
     val inputName   = inputFile.getName.replaceAll(ScalaFileExtension, "")
     val definitions = ScalaParser.parse(inputFile.parse[Source].get, inputName)
     generateFrom(definitions).map(output => s"$outputSubdir/$inputName$fileExtension" -> output)

modules/idlgen/core/src/main/scala/avro/AvroSrcGenerator.scala

@@ -46,33 +46,50 @@ object AvroSrcGenerator extends SrcGenerator {
 
   // We must process all inputs including imported files from outside our initial fileset,
   // so we then reduce our output to that based on this fileset
-  override def generateFrom(files: Set[File], options: String*): Seq[(File, String, Seq[String])] =
-    super.generateFrom(files, options: _*).filter(output => files.contains(output._1))
-
-  def generateFrom(inputFile: File, options: String*): Option[(String, Seq[String])] =
-    generateFrom(
+  override def generateFrom(
+      files: Set[File],
+      serializationType: String,
+      options: String*): Seq[(File, String, Seq[String])] =
+    super
+      .generateFrom(files, serializationType: String, options: _*)
+      .filter(output => files.contains(output._1))
+
+  def generateFrom(
+      inputFile: File,
+      serializationType: String,
+      options: String*): Option[(String, Seq[String])] =
+    generateFromSchemaProtocols(
       mainGenerator.fileParser
         .getSchemaOrProtocols(inputFile, mainGenerator.format, mainGenerator.classStore),
+      serializationType,
       options)
 
-  def generateFrom(input: String, options: String*): Option[(String, Seq[String])] =
-    generateFrom(
+  def generateFrom(
+      input: String,
+      serializationType: String,
+      options: String*): Option[(String, Seq[String])] =
+    generateFromSchemaProtocols(
       mainGenerator.stringParser
         .getSchemaOrProtocols(input, mainGenerator.schemaStore),
+      serializationType,
       options)
 
-  private def generateFrom(
+  private def generateFromSchemaProtocols(
       schemasOrProtocols: List[Either[Schema, Protocol]],
+      serializationType: String,
       options: Seq[String]): Option[(String, Seq[String])] =
     Some(schemasOrProtocols)
       .filter(_.nonEmpty)
       .flatMap(_.last match {
         case Right(p) => Some(p)
         case _        => None
       })
-      .map(generateFrom(_, options))
+      .map(generateFrom(_, serializationType, options))
 
-  def generateFrom(protocol: Protocol, options: Seq[String]): (String, Seq[String]) = {
+  def generateFrom(
+      protocol: Protocol,
+      serializationType: String,
+      options: Seq[String]): (String, Seq[String]) = {
 
     val outputPath =
       s"${protocol.getNamespace.replace('.', '/')}/${protocol.getName}$ScalaFileExtension"
@@ -93,7 +110,7 @@ object AvroSrcGenerator extends SrcGenerator {
     val messageLines = schemaLines.tail.map(line =>
       if (line.contains("case class")) s"@message $line" else line) :+ "" // note: can be "final case class"
 
-    val rpcAnnotation = s"  @rpc(${("Avro" +: options).mkString(", ")})"
+    val rpcAnnotation = s"  @rpc(${(serializationType +: options).mkString(", ")})"
     val requestLines = protocol.getMessages.asScala.toSeq.flatMap {
       case (name, message) =>
         val comment = Seq(Option(message.getDoc).map(doc => s"  /** $doc */")).flatten

modules/idlgen/core/src/main/scala/idlgen.scala

@@ -18,6 +18,7 @@ package freestyle.rpc
 
 import freestyle.rpc.idlgen.avro._
 import freestyle.rpc.idlgen.proto.ProtoIdlGenerator
+import freestyle.rpc.protocol.{Avro, AvroWithSchema, Protobuf, SerializationType}
 
 package object idlgen {
 
@@ -33,4 +34,7 @@ package object idlgen {
   val srcGenerators: Map[String, SrcGenerator] = Seq(AvroSrcGenerator)
     .map(g => g.idlType -> g)
     .toMap
+
+  val serializationTypes: Map[String, SerializationType] =
+    Map("Protobuf" -> Protobuf, "Avro" -> Avro, "AvroWithSchema" -> AvroWithSchema)
 }

modules/idlgen/core/src/test/resources/avro/MyGreeterWithSchemaService.scala

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017-2018 47 Degrees, LLC. <http://www.47deg.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package foo.bar
+
+import freestyle.rpc.protocol._
+
+@message case class HelloRequest(arg1: String, arg2: Option[String], arg3: List[String])
+
+@message case class HelloResponse(arg1: String, arg2: Option[String], arg3: List[String])
+
+@service trait MyGreeterService[F[_]] {
+
+  @rpc(AvroWithSchema, Gzip)
+  def sayHelloAvro(arg: foo.bar.HelloRequest): F[foo.bar.HelloResponse]
+
+  @rpc(AvroWithSchema, Gzip)
+  def sayNothingAvro(arg: Empty.type): F[Empty.type]
+
+}

modules/idlgen/core/src/test/scala/SrcGenTests.scala

@@ -33,16 +33,25 @@ class SrcGenTests extends RpcBaseTestSuite {
         "/avro/GreeterService.avdl",
         "/avro/MyGreeterService.scala",
         "foo/bar/MyGreeterService.scala")
+
+    "generate correct Scala classes from .avdl for AvroWithSchema serialization type" in
+      test(
+        "/avro/GreeterService.avdl",
+        "/avro/MyGreeterWithSchemaService.scala",
+        "foo/bar/MyGreeterService.scala",
+        "AvroWithSchema")
   }
 
   private def test(
       inputResourcePath: String,
       outputResourcePath: String,
-      outputFilePath: String): Unit = {
+      outputFilePath: String,
+      serializationType: String = "Avro"): Unit = {
     val expectedOutput = resource(outputResourcePath).getLines.toList
       .dropWhile(line => line.startsWith("/*") || line.startsWith(" *"))
       .tail
-    val output = AvroSrcGenerator.generateFrom(resource(inputResourcePath).mkString, "Gzip")
+    val output =
+      AvroSrcGenerator.generateFrom(resource(inputResourcePath).mkString, serializationType, "Gzip")
     output should not be empty
     val (filePath, contents) = output.get
     filePath shouldBe outputFilePath

modules/idlgen/plugin/src/main/scala/IdlGenPlugin.scala

@@ -48,6 +48,12 @@ object IdlGenPlugin extends AutoPlugin {
         "The IDL target directory, where the `idlGen` task will write the generated files " +
           "in subdirectories such as `proto` for Protobuf and `avro` for Avro, based on freestyle-rpc service definitions.")
 
+    lazy val srcGenSerializationType: SettingKey[String] =
+      settingKey[String](
+        "The serialization type when generating Scala sources from the IDL definitions." +
+          "Protobuf, Avro or AvroWithSchema are the current supported serialization types. " +
+          "By default, the serialization type is 'Avro'.")
+
     lazy val srcGenSourceDir: SettingKey[File] =
       settingKey[File]("The IDL directory, where your IDL definitions are placed.")
 
@@ -72,8 +78,9 @@ object IdlGenPlugin extends AutoPlugin {
     idlType := "(missing arg)",
     idlGenSourceDir := (Compile / sourceDirectory).value,
     idlGenTargetDir := (Compile / resourceManaged).value,
-    srcGenSourceDir := (Compile / resourceDirectory).value,
+    srcGenSerializationType := "Avro",
     srcJarNames := Seq.empty,
+    srcGenSourceDir := (Compile / resourceDirectory).value,
     srcGenTargetDir := (Compile / sourceManaged).value,
     genOptions := Seq.empty
   )
@@ -83,12 +90,14 @@ object IdlGenPlugin extends AutoPlugin {
       idlGen := idlGenTask(
         IdlGenApplication,
         idlType.value,
+        srcGenSerializationType.value,
         genOptions.value,
         idlGenTargetDir.value,
         target.value / "idlGen")(idlGenSourceDir.value.allPaths.get.toSet).toSeq,
       srcGen := idlGenTask(
         SrcGenApplication,
         idlType.value,
+        srcGenSerializationType.value,
         genOptions.value,
         srcGenTargetDir.value,
         target.value / "srcGen")(srcGenSourceDir.value.allPaths.get.toSet).toSeq,
@@ -106,12 +115,13 @@ object IdlGenPlugin extends AutoPlugin {
   private def idlGenTask(
       generator: GeneratorApplication[_],
       idlType: String,
+      serializationType: String,
       options: Seq[String],
       targetDir: File,
       cacheDir: File): Set[File] => Set[File] =
     FileFunction.cached(cacheDir, FilesInfo.lastModified, FilesInfo.exists) {
       (inputFiles: Set[File]) =>
-        generator.generateFrom(idlType, inputFiles, targetDir, options: _*).toSet
+        generator.generateFrom(idlType, serializationType, inputFiles, targetDir, options: _*).toSet
     }
 
   private def extractIDLDefinitionsFromJar(

modules/idlgen/plugin/src/sbt-test/sbt-frees-rpc-idlgen/avroWithSchema/build.sbt

@@ -0,0 +1,7 @@
+version := sys.props("version")
+
+resolvers += Resolver.bintrayRepo("beyondthelines", "maven")
+
+libraryDependencies ++= Seq(
+  "io.frees" %% "frees-rpc-server" % sys.props("version")
+)

modules/idlgen/plugin/src/sbt-test/sbt-frees-rpc-idlgen/avroWithSchema/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.1.2

modules/idlgen/plugin/src/sbt-test/sbt-frees-rpc-idlgen/avroWithSchema/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("io.frees" %% "sbt-frees-rpc-idlgen" % sys.props("version"))

modules/idlgen/plugin/src/sbt-test/sbt-frees-rpc-idlgen/avroWithSchema/src/main/resources/service.avdl

@@ -0,0 +1,16 @@
+@namespace("io.frees")
+protocol service {
+
+  record Foo {
+    string bar;
+  }
+
+  record Person {
+    long id;
+    string name;
+    union { null, string } email;
+  }
+
+  io.frees.Person hi(io.frees.Foo request);
+
+}

modules/idlgen/plugin/src/sbt-test/sbt-frees-rpc-idlgen/avroWithSchema/test

@@ -0,0 +1,12 @@
+$ exists src/main/resources/service.avdl
+> 'set idlType := "avro"'
+> 'set srcGenSerializationType := "Avro"'
+> srcGen
+$ exists target/scala-2.12/src_managed/main/io/frees/service.scala
+$ delete target/scala-2.12/src_managed/main/io/frees/service.scala
+
+$ exists src/main/resources/service.avdl
+> 'set idlType := "avro"'
+> 'set srcGenSerializationType := "AvroWithSchema"'
+> srcGen
+$ exists target/scala-2.12/src_managed/main/io/frees/service.scala