diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ac6bea633f..0e5794ae93 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,10 @@ New Features * util-core: Add `ClasspathResource`, a utility for loading classpath resources as an optional `InputStream`. ``PHAB_ID=D687324`` +* util-jackson: Add `com.twitter.util.jackson.YAML` for YAML serde operations with a + default configured `ScalaObjectMapper.` Add more methods to `com.twitter.util.jackson.JSON` + ``PHAB_ID=D687327`` + * util-jackson: Introduce a new library for JSON serialization and deserialization based on the Jackson integration in `Finatra `__. diff --git a/doc/src/sphinx/util-jackson/index.rst b/doc/src/sphinx/util-jackson/index.rst index 448668c89b..8d70f01f83 100644 --- a/doc/src/sphinx/util-jackson/index.rst +++ b/doc/src/sphinx/util-jackson/index.rst @@ -798,15 +798,17 @@ than the `scala.util.parsing.json.JSON` utility. .. important:: - The `c.t.util.jackson.JSON` API does not return an exception when parsing but rather returns an - `Option[T]` result. When parsing is successful, this is a `Some(T)`, otherwise it is a `None`. But - note that the specifics of any failure are lost. + The `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__ API does not return an exception when + parsing but rather returns an `Option[T]` result. When parsing is successful, this is a `Some(T)`, + otherwise it is a `None`. But note that the specifics of any failure are lost. It is thus also important to note that for this reason that the `c.t.util.jackson.JSON` uses a - default configured `ScalaObjectMapper <#defaults>`__ **with validation specifically disabled**, + `default configured ScalaObjectMapper <#defaults>`__ **with validation specifically disabled**, such that no `Bean Validation 2.0 <../util-validator>`__ style validations are performed when - parsing with `c.t.util.jackson.JSON`. Users should prefer using a configured `ScalaObjectMapper` - to perform validations in order to be able to properly handle validation exceptions. + parsing with `c.t.util.jackson.JSON`. + + Users should prefer using a configured |ScalaObjectMapper|_ to perform validations in order to + be able to properly handle validation exceptions. .. code:: scala :emphasize-lines: 7, 13, 22, 25 @@ -841,6 +843,63 @@ than the `scala.util.parsing.json.JSON` utility. "id" : "99999999" } + scala> + + +`c.t.util.jackson.YAML` +----------------------- + +Similarly, there is also a utility for `YAML `__ serde operations with many of +the same methods as `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__ using a default |ScalaObjectMapper|_ +configured with a `YAMLFactory`. + +.. code:: scala + :emphasize-lines: 8, 9, 10, 11, 12, 18, 19, 20, 29 + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> import com.twitter.util.jackson.YAML + import com.twitter.util.jackson.YAML + + scala> val result = + | YAML.parse[Map[String, Int]](""" + | |--- + | |a: 1 + | |b: 2 + | |c: 3""".stripMargin) + result: Option[Map[String,Int]] = Some(Map(a -> 1, b -> 2, c -> 3)) + + scala> case class FooClass(id: String) + defined class FooClass + + scala> val result = YAML.parse[FooClass](""" + | |--- + | |id: abcde1234""".stripMargin) + result: Option[FooClass] = Some(FooClass(abcde1234)) + + scala> result.get + res0: FooClass = FooClass(abcde1234) + + scala> val f = FooClass("99999999") + f: FooClass = FooClass(99999999) + + scala> YAML.write(f) + res1: String = + "--- + id: "99999999" + " + + scala> + +.. important:: + + Like with `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__, the `c.t.util.jackson.YAML <#c-t-util-jackson-yaml>`__ + API does not return an exception when parsing but rather returns an `Option[T]` result. + + Users should prefer using a configured |ScalaObjectMapper|_ to perform validations in order to be + able to properly handle validation exceptions. + `c.t.util.jackson.JsonDiff` --------------------------- diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD index f352f749f0..61ab5add42 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD @@ -30,6 +30,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson-dataformat-yaml", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "util/util-core/src/main/scala/com/twitter/io", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], ) diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala index 8afa7f8036..2008cbf22b 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala @@ -1,6 +1,8 @@ package com.twitter.util.jackson +import com.twitter.io.{Buf, ClasspathResource} import com.twitter.util.Try +import java.io.{File, InputStream} /** * Uses an instance of a [[ScalaObjectMapper]] configured to not perform any type of validation. @@ -18,6 +20,25 @@ object JSON { def parse[T: Manifest](input: String): Option[T] = Try(Mapper.parse[T](input)).toOption + /** Simple utility to parse a JSON [[Buf]] into an Option[T] type. */ + def parse[T: Manifest](input: Buf): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** + * Simple utility to parse a JSON [[InputStream]] into an Option[T] type. + * @note the caller is responsible for managing the lifecycle of the given [[InputStream]]. + */ + def parse[T: Manifest](input: InputStream): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** + * Simple utility to load a JSON file and parse contents into an Option[T] type. + * + * @note the caller is responsible for managing the lifecycle of the given [[File]]. + */ + def parse[T: Manifest](f: File): Option[T] = + Try(Mapper.underlying.readValue[T](f)).toOption + /** Simple utility to write a value as a JSON encoded String. */ def write(any: Any): String = Mapper.writeValueAsString(any) @@ -25,4 +46,24 @@ object JSON { /** Simple utility to pretty print a JSON encoded String from the given instance. */ def prettyPrint(any: Any): String = Mapper.writePrettyString(any) + + object Resource { + + /** + * Simple utility to load a JSON resource from the classpath and parse contents into an Option[T] type. + * + * @note `name` resolution to locate the resource is governed by [[java.lang.Class#getResourceAsStream]] + */ + def parse[T: Manifest](name: String): Option[T] = { + ClasspathResource.load(name) match { + case Some(inputStream) => + try { + JSON.parse[T](inputStream) + } finally { + inputStream.close() + } + case _ => None + } + } + } } diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/YAML.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/YAML.scala new file mode 100644 index 0000000000..a15a4aae46 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/YAML.scala @@ -0,0 +1,65 @@ +package com.twitter.util.jackson + +import com.twitter.io.{Buf, ClasspathResource} +import com.twitter.util.Try +import java.io.{File, InputStream} + +/** + * Uses an instance of a [[ScalaObjectMapper]] configured to not perform any type of validation. + * Inspired by [[https://www.scala-lang.org/api/2.12.6/scala-parser-combinators/scala/util/parsing/json/JSON$.html]]. + * + * @note This is only intended for use from Scala (not Java). + * + * @see [[ScalaObjectMapper]] + */ +object YAML { + private final val Mapper: ScalaObjectMapper = + ScalaObjectMapper.builder.withNoValidation.yamlObjectMapper + + /** Simple utility to parse a YAML string into an Option[T] type. */ + def parse[T: Manifest](input: String): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** Simple utility to parse a YAML [[Buf]] into an Option[T] type. */ + def parse[T: Manifest](input: Buf): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** + * Simple utility to parse a YAML [[InputStream]] into an Option[T] type. + * @note the caller is responsible for managing the lifecycle of the given [[InputStream]]. + */ + def parse[T: Manifest](input: InputStream): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** + * Simple utility to load a YAML file and parse contents into an Option[T] type. + * + * @note the caller is responsible for managing the lifecycle of the given [[File]]. + */ + def parse[T: Manifest](f: File): Option[T] = + Try(Mapper.underlying.readValue[T](f)).toOption + + /** Simple utility to write a value as a YAML encoded String. */ + def write(any: Any): String = + Mapper.writeValueAsString(any) + + object Resource { + + /** + * Simple utility to load a YAML resource from the classpath and parse contents into an Option[T] type. + * + * @note `name` resolution to locate the resource is governed by [[java.lang.Class#getResourceAsStream]] + */ + def parse[T: Manifest](name: String): Option[T] = { + ClasspathResource.load(name) match { + case Some(inputStream) => + try { + YAML.parse[T](inputStream) + } finally { + inputStream.close() + } + case _ => None + } + } + } +} diff --git a/util-jackson/src/test/resources/BUILD b/util-jackson/src/test/resources/BUILD new file mode 100644 index 0000000000..a001371a96 --- /dev/null +++ b/util-jackson/src/test/resources/BUILD @@ -0,0 +1,3 @@ +resources( + sources = ["**/*"], +) diff --git a/util-jackson/src/test/resources/test.json b/util-jackson/src/test/resources/test.json new file mode 100644 index 0000000000..98e5dbd51e --- /dev/null +++ b/util-jackson/src/test/resources/test.json @@ -0,0 +1,3 @@ +{ + "id": "55555555" +} diff --git a/util-jackson/src/test/resources/test.yml b/util-jackson/src/test/resources/test.yml new file mode 100644 index 0000000000..ac1f85b4b0 --- /dev/null +++ b/util-jackson/src/test/resources/test.yml @@ -0,0 +1,2 @@ +--- +id: 55555555 diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD index fdeba22589..b630aac594 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD @@ -33,6 +33,7 @@ junit_tests( "util/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions", "util/util-jackson/src/main/scala/com/twitter/util/jackson/serde", "util/util-jackson/src/test/java/com/twitter/util/jackson", + "util/util-jackson/src/test/resources", "util/util-mock/src/main/scala/com/twitter/util/mock", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", "util/util-validator/src/main/java/com/twitter/util/validation", diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/FileResources.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/FileResources.scala new file mode 100644 index 0000000000..d68f8fc6b5 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/FileResources.scala @@ -0,0 +1,29 @@ +package com.twitter.util.jackson + +import com.twitter.io.TempFolder +import java.io.{FileOutputStream, OutputStream, File => JFile} +import java.nio.charset.StandardCharsets +import java.nio.file.{FileSystems, Files} + +trait FileResources extends TempFolder { + + protected def writeStringToFile( + directory: String, + name: String, + ext: String, + data: String + ): JFile = { + val file = Files.createTempFile(FileSystems.getDefault.getPath(directory), name, ext).toFile + try { + val out: OutputStream = new FileOutputStream(file, false) + out.write(data.getBytes(StandardCharsets.UTF_8)) + file + } finally { + file.deleteOnExit() + } + } + + protected def addSlash(directory: String): String = { + if (directory.endsWith("/")) directory else s"$directory/" + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala index 9c0e19953a..c7af26b123 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala @@ -1,29 +1,26 @@ package com.twitter.util.jackson +import com.twitter.io.Buf +import java.io.{ByteArrayInputStream, File => JFile} import org.junit.runner.RunWith import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatestplus.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) -class JSONTest extends AnyFunSuite with Matchers { +class JSONTest extends AnyFunSuite with Matchers with FileResources { test("JSON#parse 1") { JSON.parse[Map[String, Int]]("""{"a": 1, "b": 2}""") match { case Some(map) => - map("a") should equal(1) - map("b") should equal(2) + map should be(Map("a" -> 1, "b" -> 2)) case _ => fail() } } test("JSON#parse 2") { JSON.parse[Seq[String]]("""["a", "b", "c"]""") match { - case Some(seq) => - seq.size should equal(3) - seq.head should be("a") - seq(1) should be("b") - seq.last should be("c") + case Some(Seq("a", "b", "c")) => // pass case _ => fail() } } @@ -46,6 +43,40 @@ class JSONTest extends AnyFunSuite with Matchers { } } + test("JSON#parse 5") { + val buf = Buf.Utf8("""{"id": "abcd1234"}""") + JSON.parse[FooClass](buf) match { + case Some(foo) => + foo.id should equal("abcd1234") + case _ => fail() + } + } + + test("JSON#parse 6") { + val inputStream = new ByteArrayInputStream("""{"id": "abcd1234"}""".getBytes("UTF-8")) + try { + JSON.parse[FooClass](inputStream) match { + case Some(foo) => + foo.id should equal("abcd1234") + case _ => fail() + } + } finally { + inputStream.close() + } + } + + test("JSON#parse 7") { + withTempFolder { + val file: JFile = + writeStringToFile(folderName, "test-file", ".json", """{"id": "999999999"}""") + JSON.parse[FooClass](file) match { + case Some(foo) => + foo.id should equal("999999999") + case _ => fail() + } + } + } + test("JSON#write 1") { JSON.write(Map("a" -> 1, "b" -> 2)) should equal("""{"a":1,"b":2}""") } @@ -78,4 +109,12 @@ class JSONTest extends AnyFunSuite with Matchers { | "id" : "abcd1234" |}""".stripMargin) } + + test("JSON.Resource#parse resource") { + JSON.Resource.parse[FooClass]("/test.json") match { + case Some(foo) => + foo.id should equal("55555555") + case _ => fail() + } + } } diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/YAMLTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/YAMLTest.scala new file mode 100644 index 0000000000..79d9b16f25 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/YAMLTest.scala @@ -0,0 +1,159 @@ +package com.twitter.util.jackson + +import com.twitter.io.Buf +import java.io.{ByteArrayInputStream, File => JFile} +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class YAMLTest extends AnyFunSuite with Matchers with FileResources { + + test("YAML#parse 1") { + YAML.parse[Map[String, Int]](""" + |--- + |a: 1 + |b: 2 + |c: 3""".stripMargin) match { + case Some(map) => + map should be(Map("a" -> 1, "b" -> 2, "c" -> 3)) + case _ => fail() + } + } + + test("YAML#parse 2") { + // without quotes + YAML.parse[Seq[String]](""" + |--- + |- a + |- b + |- c""".stripMargin) match { + case Some(Seq("a", "b", "c")) => // pass + case _ => fail() + } + // with quotes + YAML.parse[Seq[String]](""" + |--- + |- "a" + |- "b" + |- "c"""".stripMargin) match { + case Some(Seq("a", "b", "c")) => // pass + case _ => fail() + } + } + + test("YAML#parse 3") { + // without quotes + YAML.parse[FooClass](""" + |--- + |id: abcde1234""".stripMargin) match { + case Some(foo) => + foo.id should equal("abcde1234") + case _ => fail() + } + // with quotes + YAML.parse[FooClass](""" + |--- + |id: "abcde1234"""".stripMargin) match { + case Some(foo) => + foo.id should equal("abcde1234") + case _ => fail() + } + } + + test("YAML#parse 4") { + YAML.parse[CaseClassWithNotEmptyValidation](""" + |--- + |name: '' + |make: vw""".stripMargin) match { + case Some(clazz) => + // performs no validations + clazz.name.isEmpty should be(true) + clazz.make should equal(CarMakeEnum.vw) + case _ => fail() + } + } + + test("YAML#parse 5") { + val buf = Buf.Utf8("""{"id": "abcd1234"}""") + YAML.parse[FooClass](buf) match { + case Some(foo) => + foo.id should equal("abcd1234") + case _ => fail() + } + } + + test("YAML#parse 6") { + val inputStream = new ByteArrayInputStream("""{"id": "abcd1234"}""".getBytes("UTF-8")) + try { + YAML.parse[FooClass](inputStream) match { + case Some(foo) => + foo.id should equal("abcd1234") + case _ => fail() + } + } finally { + inputStream.close() + } + } + + test("YAML#parse 7") { + withTempFolder { + // without quotes + val file1: JFile = + writeStringToFile( + folderName, + "test-file-quotes", + ".yaml", + """|--- + |id: 999999999""".stripMargin) + YAML.parse[FooClass](file1) match { + case Some(foo) => + foo.id should equal("999999999") + case _ => fail() + } + // with quotes + val file2: JFile = + writeStringToFile( + folderName, + "test-file-noquotes", + ".yaml", + """|--- + |id: "999999999"""".stripMargin) + YAML.parse[FooClass](file2) match { + case Some(foo) => + foo.id should equal("999999999") + case _ => fail() + } + } + } + + test("YAML#write 1") { + YAML.write(Map("a" -> 1, "b" -> 2)) should equal("""|--- + |a: 1 + |b: 2 + |""".stripMargin) + } + + test("YAML#write 2") { + YAML.write(Seq("a", "b", "c")) should equal("""|--- + |- "a" + |- "b" + |- "c" + |""".stripMargin) + } + + test("YAML#write 3") { + YAML.write(FooClass("abcd1234")) should equal("""|--- + |id: "abcd1234" + |""".stripMargin) + } + + test("YAML.Resource#parse resource") { + YAML.Resource.parse[FooClass]("/test.yml") match { + case Some(foo) => + foo.id should equal("55555555") + case _ => fail() + } + } +}