Codegen: serialize empty arrays and require them (jsoniter fix) (#3655)

.github/workflows/ci.yml

@@ -77,7 +77,14 @@ jobs:
       if: matrix.target-platform == 'JVM' && matrix.java == '11'
       run: sbt $SBT_JAVA_OPTS -v compileDocumentation
     - name: Test
-      if: matrix.target-platform != 'JS'
+      if: matrix.target-platform == 'JVM' && matrix.scala-version == '2.12'
+      uses: nick-fields/retry@v2
+      with:
+        timeout_minutes: 15
+        max_attempts: 4
+        command: sbt $SBT_JAVA_OPTS -v "testScoped ${{ matrix.scala-version }} ${{ matrix.target-platform }}; openapiCodegenSbt2_12/scripted"
+    - name: Test
+      if: matrix.target-platform != 'JS' && !(matrix.target-platform == 'JVM' && matrix.scala-version == '2.12')
       uses: nick-fields/retry@v2
       with:
         timeout_minutes: 15

openapi-codegen/core/src/main/scala/sttp/tapir/codegen/JsonSerdeGenerator.scala

@@ -197,7 +197,12 @@ object JsonSerdeGenerator {
   private val jsoniterPkgRoot = "com.github.plokhotnyuk.jsoniter_scala"
   private val jsoniterPkgCore = s"$jsoniterPkgRoot.core"
   private val jsoniterPkgMacros = s"$jsoniterPkgRoot.macros"
-  private val jsoniterBaseConfig = s"$jsoniterPkgMacros.CodecMakerConfig.withAllowRecursiveTypes(true)"
+  // By default:
+  // - permit recursive schema definitions
+  // - force serialization of empty collections if 'required' (non-required T will be typed as 'Option[T]' to which this will not apply)
+  // - require presence of collections when decoding if 'required'
+  private val jsoniterBaseConfig =
+    s"$jsoniterPkgMacros.CodecMakerConfig.withAllowRecursiveTypes(true).withTransientEmpty(false).withRequireCollectionFields(true)"
   private val jsoniteEnumConfig = s"$jsoniterBaseConfig.withDiscriminatorFieldName(scala.None)"
   private def genJsoniterSerdes(
       doc: OpenapiDocument,

openapi-codegen/core/src/test/scala/sttp/tapir/codegen/ClassDefinitionGeneratorSpec.scala

@@ -490,7 +490,7 @@ class ClassDefinitionGeneratorSpec extends CompileCheckTestBase {
       res shouldCompile ()
       fullRes shouldCompile ()
       extra.get should include(
-        """implicit lazy val reqWithVariantsCodec: com.github.plokhotnyuk.jsoniter_scala.core.JsonValueCodec[ReqWithVariants] = com.github.plokhotnyuk.jsoniter_scala.macros.JsonCodecMaker.make(com.github.plokhotnyuk.jsoniter_scala.macros.CodecMakerConfig.withAllowRecursiveTypes(true).withRequireDiscriminatorFirst(false).withDiscriminatorFieldName(Some("type")))"""
+        """implicit lazy val reqWithVariantsCodec: com.github.plokhotnyuk.jsoniter_scala.core.JsonValueCodec[ReqWithVariants] = com.github.plokhotnyuk.jsoniter_scala.macros.JsonCodecMaker.make(com.github.plokhotnyuk.jsoniter_scala.macros.CodecMakerConfig.withAllowRecursiveTypes(true).withTransientEmpty(false).withRequireCollectionFields(true).withRequireDiscriminatorFirst(false).withDiscriminatorFieldName(Some("type")))"""
       )
     }
     testOK(TestHelpers.oneOfDocsWithMapping)

openapi-codegen/sbt-plugin/README.md

@@ -13,5 +13,5 @@ scripted
 cd sbt/sbt-openapi-codegen/src/sbt-test/sbt-openapi-codegen/minimal/
 sbt -Dplugin.version=0.1-SNAPSHOT run
 cat target/swagger.yaml
-cat target/scala-2.12/classes/sttp/tapir/generated/TapirGeneratedEndpoints.scala
+cat target/scala-2.13/classes/sttp/tapir/generated/TapirGeneratedEndpoints.scala
 ```

openapi-codegen/sbt-plugin/src/main/scala/sttp/tapir/sbt/OpenapiCodegenTask.scala

@@ -33,7 +33,7 @@ case class OpenapiCodegenTask(
   }
 
   def cachedCopyFile(tempFile: File, outFile: File) =
-    inputChanged(cacheDir / "sbt-openapi-codegen-inputs") { (inChanged, _: HashFileInfo) =>
+    inputChanged(cacheDir / s"sbt-openapi-codegen-inputs-${tempFile.getName}") { (inChanged, _: HashFileInfo) =>
       if (inChanged || !outFile.exists) {
         IO.copyFile(tempFile, outFile, preserveLastModified = true)
       }

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/caching/build.sbt

@@ -1,13 +1,13 @@
 lazy val root = (project in file("."))
   .enablePlugins(OpenapiCodegenPlugin)
   .settings(
-    scalaVersion := "2.12.4",
+    scalaVersion := "2.13.13",
     version := "0.1"
   )
 
-libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-json-circe" % "0.17.0-M2"
-libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-openapi-docs" % "0.17.0-M2"
-libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-openapi-circe-yaml" % "0.17.0-M2"
+libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-json-circe" % "1.10.0"
+libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-openapi-docs" % "1.10.0"
+libraryDependencies += "com.softwaremill.sttp.apispec" %% "openapi-circe-yaml" % "0.8.0"
 
 import scala.io.Source
 

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/caching/src/main/scala/Main.scala

@@ -22,7 +22,7 @@ object Main extends App {
   import sttp.tapir.generated._
   import sttp.tapir.docs.openapi._
 
-  val docs = TapirGeneratedEndpoints.generatedEndpoints.toOpenAPI("My Bookshop", "1.0")
+  val docs = OpenAPIDocsInterpreter().toOpenAPI(TapirGeneratedEndpoints.generatedEndpoints, "My Bookshop", "1.0")
 
   import java.nio.file.{Paths, Files}
   import java.nio.charset.StandardCharsets

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/caching/swagger.yaml

@@ -17,12 +17,14 @@ paths:
         required: true
         schema:
           type: integer
+          format: int32
       - name: limit
         in: query
         description: Maximum number of books to retrieve
         required: true
         schema:
           type: integer
+          format: int32
       - name: X-Auth-Token
         in: header
         required: true
@@ -37,7 +39,7 @@ paths:
                 type: array
                 items:
                   $ref: '#/components/schemas/Book'
-        default:
+        '400':
           description: ''
           content:
             text/plain:
@@ -46,6 +48,7 @@ paths:
 components:
   schemas:
     Book:
+      title: Book
       required:
       - title
       type: object

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/caching/test

@@ -1,6 +1,6 @@
 > clean
 > run
 > check
-$ copy-file target/scala-2.12/src_managed/main/sbt-openapi-codegen/TapirGeneratedEndpoints.scala target/TapirGeneratedEndpoints.scala
+$ copy-file target/scala-2.13/src_managed/main/sbt-openapi-codegen/TapirGeneratedEndpoints.scala target/TapirGeneratedEndpoints.scala
 > compile
-$ newer target/TapirGeneratedEndpoints.scala target/scala-2.12/src_managed/main/sbt-openapi-codegen/TapirGeneratedEndpoints.scala
+$ newer target/TapirGeneratedEndpoints.scala target/scala-2.13/src_managed/main/sbt-openapi-codegen/TapirGeneratedEndpoints.scala

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/minimal/build.sbt

@@ -1,13 +1,13 @@
 lazy val root = (project in file("."))
   .enablePlugins(OpenapiCodegenPlugin)
   .settings(
-    scalaVersion := "2.12.4",
+    scalaVersion := "2.13.13",
     version := "0.1"
   )
 
-libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-json-circe" % "0.17.0-M2"
-libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-openapi-docs" % "0.17.0-M2"
-libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-openapi-circe-yaml" % "0.17.0-M2"
+libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-json-circe" % "1.10.0"
+libraryDependencies += "com.softwaremill.sttp.tapir" %% "tapir-openapi-docs" % "1.10.0"
+libraryDependencies += "com.softwaremill.sttp.apispec" %% "openapi-circe-yaml" % "0.8.0"
 
 import scala.io.Source
 

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/minimal/src/main/scala/Main.scala

@@ -22,7 +22,7 @@ object Main extends App {
   import sttp.tapir.generated._
   import sttp.tapir.docs.openapi._
 
-  val docs = TapirGeneratedEndpoints.generatedEndpoints.toOpenAPI("My Bookshop", "1.0")
+  val docs = OpenAPIDocsInterpreter().toOpenAPI(TapirGeneratedEndpoints.generatedEndpoints, "My Bookshop", "1.0")
 
   import java.nio.file.{Paths, Files}
   import java.nio.charset.StandardCharsets

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/minimal/swagger.yaml

@@ -17,12 +17,14 @@ paths:
         required: true
         schema:
           type: integer
+          format: int32
       - name: limit
         in: query
         description: Maximum number of books to retrieve
         required: true
         schema:
           type: integer
+          format: int32
       - name: X-Auth-Token
         in: header
         required: true
@@ -37,7 +39,7 @@ paths:
                 type: array
                 items:
                   $ref: '#/components/schemas/Book'
-        default:
+        '400':
           description: ''
           content:
             text/plain:
@@ -46,6 +48,7 @@ paths:
 components:
   schemas:
     Book:
+      title: Book
       required:
       - title
       type: object

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/oneOf-json-roundtrip/Expected.scala.txt

@@ -0,0 +1,61 @@
+package sttp.tapir.generated
+
+object TapirGeneratedEndpoints {
+
+  import sttp.tapir._
+  import sttp.tapir.model._
+  import sttp.tapir.generic.auto._
+  import sttp.tapir.json.circe._
+  import io.circe.generic.semiauto._
+
+  import sttp.tapir.generated.TapirGeneratedEndpointsJsonSerdes._
+
+  sealed trait ADTWithoutDiscriminator
+  sealed trait ADTWithDiscriminator
+  sealed trait ADTWithDiscriminatorNoMapping
+  case class SubtypeWithoutD1 (
+    s: String,
+    i: Option[Int] = None,
+    a: Seq[String],
+    absent: Option[String] = None
+  ) extends ADTWithoutDiscriminator
+  case class SubtypeWithD1 (
+    s: String,
+    i: Option[Int] = None,
+    d: Option[Double] = None
+  ) extends ADTWithDiscriminator with ADTWithDiscriminatorNoMapping
+  case class SubtypeWithoutD3 (
+    s: String,
+    i: Option[Int] = None,
+    d: Option[Double] = None,
+    absent: Option[String] = None
+  ) extends ADTWithoutDiscriminator
+  case class SubtypeWithoutD2 (
+    a: Seq[String],
+    absent: Option[String] = None
+  ) extends ADTWithoutDiscriminator
+  case class SubtypeWithD2 (
+    s: String,
+    a: Option[Seq[String]] = None
+  ) extends ADTWithDiscriminator with ADTWithDiscriminatorNoMapping
+
+
+
+  lazy val putAdtTest =
+    endpoint
+      .put
+      .in(("adt" / "test"))
+      .in(jsonBody[ADTWithoutDiscriminator])
+      .out(jsonBody[ADTWithoutDiscriminator].description("successful operation"))
+
+  lazy val postAdtTest =
+    endpoint
+      .post
+      .in(("adt" / "test"))
+      .in(jsonBody[ADTWithDiscriminatorNoMapping])
+      .out(jsonBody[ADTWithDiscriminator].description("successful operation"))
+
+
+  lazy val generatedEndpoints = List(putAdtTest, postAdtTest)
+
+}

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/oneOf-json-roundtrip/build.sbt

@@ -0,0 +1,34 @@
+lazy val root = (project in file("."))
+  .enablePlugins(OpenapiCodegenPlugin)
+  .settings(
+    scalaVersion := "2.13.13",
+    version := "0.1"
+  )
+
+libraryDependencies ++= Seq(
+  "com.softwaremill.sttp.tapir" %% "tapir-json-circe" % "1.10.0",
+  "com.softwaremill.sttp.tapir" %% "tapir-openapi-docs" % "1.10.0",
+  "com.softwaremill.sttp.apispec" %% "openapi-circe-yaml" % "0.8.0",
+  "io.circe" %% "circe-generic" % "0.14.6",
+  "com.beachape" %% "enumeratum" % "1.7.3",
+  "com.beachape" %% "enumeratum-circe" % "1.7.3",
+  "org.scalatest" %% "scalatest" % "3.2.18" % Test,
+  "com.softwaremill.sttp.tapir" %% "tapir-sttp-stub-server" % "1.10.0" % Test
+)
+
+import scala.io.Source
+
+TaskKey[Unit]("check") := {
+  val generatedCode =
+    Source.fromFile("target/scala-2.13/src_managed/main/sbt-openapi-codegen/TapirGeneratedEndpoints.scala").getLines.mkString("\n")
+  val expected = Source.fromFile("Expected.scala.txt").getLines.mkString("\n")
+  val generatedTrimmed = generatedCode.linesIterator.zipWithIndex.filterNot(_._1.forall(_.isWhitespace)).map{ case (a, i) => a.trim -> i }.toSeq
+  val expectedTrimmed = expected.linesIterator.filterNot(_.forall(_.isWhitespace)).map(_.trim).toSeq
+  if (generatedTrimmed.size != expectedTrimmed.size)
+    sys.error(s"expected ${expectedTrimmed.size} non-empty lines, found ${generatedTrimmed.size}")
+  generatedTrimmed.zip(expectedTrimmed).foreach { case ((a, i), b) =>
+    if (a != b) sys.error(s"Generated code did not match (expected '$b' on line $i, found '$a')")
+  }
+  println("Skipping swagger roundtrip for petstore")
+  ()
+}

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/oneOf-json-roundtrip/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.9.9

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/oneOf-json-roundtrip/project/plugins.sbt

@@ -0,0 +1,11 @@
+{
+  val pluginVersion = System.getProperty("plugin.version")
+  if (pluginVersion == null)
+    throw new RuntimeException("""|
+                                  |
+                                  |The system property 'plugin.version' is not defined.
+                                  |Specify this property using the scriptedLaunchOpts -D.
+                                  |
+                                  |""".stripMargin)
+  else addSbtPlugin("com.softwaremill.sttp.tapir" % "sbt-openapi-codegen" % pluginVersion)
+}

openapi-codegen/sbt-plugin/src/sbt-test/sbt-openapi-codegen/oneOf-json-roundtrip/src/main/scala/Main.scala

@@ -0,0 +1,31 @@
+object Main extends App {
+  /*
+  import sttp.tapir._
+  import sttp.tapir.json.circe._
+  import io.circe.generic.auto._
+
+  type Limit = Int
+  type AuthToken = String
+  case class BooksFromYear(genre: String, year: Int)
+  case class Book(title: String)
+
+  val booksListing: Endpoint[(BooksFromYear, Limit, AuthToken), String, List[Book], Any] =
+    endpoint
+      .get
+      .in(("books" / path[String]("genre") / path[Int]("year")).mapTo[BooksFromYear])
+      .in(query[Limit]("limit").description("Maximum number of books to retrieve"))
+      .in(header[AuthToken]("X-Auth-Token"))
+      .errorOut(stringBody)
+      .out(jsonBody[List[Book]])
+   */
+  import sttp.apispec.openapi.circe.yaml._
+  import sttp.tapir.generated._
+  import sttp.tapir.docs.openapi._
+
+  val docs = OpenAPIDocsInterpreter().toOpenAPI(TapirGeneratedEndpoints.generatedEndpoints, "My Bookshop", "1.0")
+
+  import java.nio.file.{Paths, Files}
+  import java.nio.charset.StandardCharsets
+
+  Files.write(Paths.get("target/swagger.yaml"), docs.toYaml.getBytes(StandardCharsets.UTF_8))
+}