From b74cafe3e32566d4d8331e6efa0a3a24a12f8e72 Mon Sep 17 00:00:00 2001 From: Kamil Podsiadlo Date: Fri, 12 Nov 2021 20:22:18 +0100 Subject: [PATCH 1/3] Add Tag property to Nodes --- .../scala/org/virtuslab/yaml/TestRunner.scala | 3 +- .../main/scala/org/virtuslab/yaml/Node.scala | 68 +++++++++++++------ .../main/scala/org/virtuslab/yaml/Tag.scala | 42 ++++++++++++ .../org/virtuslab/yaml/YamlDecoder.scala | 31 ++++----- .../org/virtuslab/yaml/YamlEncoder.scala | 28 ++++---- .../dump/serialize/SerializerImpl.scala | 6 +- .../yaml/internal/load/compose/Composer.scala | 22 +++--- .../yaml/internal/load/parse/Event.scala | 5 +- .../yaml/internal/load/parse/ParserImpl.scala | 1 + .../org/virtuslab/yaml/ComposerSuite.scala | 32 ++++----- .../org/virtuslab/yaml/ConstructSuite.scala | 10 +-- .../org/virtuslab/yaml/parser/TagSuite.scala | 2 +- .../virtuslab/yaml/tokenizer/TagSuite.scala | 1 - 13 files changed, 158 insertions(+), 93 deletions(-) create mode 100644 yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala diff --git a/tests/test-suite/jvm/src/main/scala/org/virtuslab/yaml/TestRunner.scala b/tests/test-suite/jvm/src/main/scala/org/virtuslab/yaml/TestRunner.scala index d0aca8798..2e3c33032 100644 --- a/tests/test-suite/jvm/src/main/scala/org/virtuslab/yaml/TestRunner.scala +++ b/tests/test-suite/jvm/src/main/scala/org/virtuslab/yaml/TestRunner.scala @@ -6,13 +6,12 @@ import scala.util.Failure import scala.util.Success import scala.util.Try -import org.virtuslab.yaml +import org.virtuslab.yaml.Tag import org.virtuslab.yaml.internal.load.parse.Anchor import org.virtuslab.yaml.internal.load.parse.EventKind import org.virtuslab.yaml.internal.load.parse.EventKind.* import org.virtuslab.yaml.internal.load.parse.NodeEventMetadata import org.virtuslab.yaml.internal.load.parse.ParserImpl -import org.virtuslab.yaml.internal.load.parse.Tag import org.virtuslab.yaml.internal.load.reader.Scanner import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala index e9ec7cfb4..9aa467ed9 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala @@ -1,46 +1,76 @@ package org.virtuslab.yaml import org.virtuslab.yaml.Range +import org.virtuslab.yaml.Tag import org.virtuslab.yaml.syntax.YamlPrimitive /** * ADT that corresponds to the YAML representation graph nodes https://yaml.org/spec/1.2/spec.html#id2764044 */ sealed trait Node: - def pos: Option[Range] + private[yaml] def pos: Option[Range] + def tag: Tag object Node: - final case class ScalarNode(value: String, pos: Option[Range] = None) extends Node + final case class ScalarNode private[yaml] (value: String, tag: Tag, pos: Option[Range] = None) + extends Node object ScalarNode: - def apply(value: String): ScalarNode = new ScalarNode(value) + def apply(value: String): ScalarNode = new ScalarNode(value, Tag.resolveTag(value)) + def unapply(node: ScalarNode): Option[(String, Tag)] = Some((node.value, node.tag)) + end ScalarNode - final case class SequenceNode(nodes: Seq[Node], pos: Option[Range] = None) extends Node + final case class SequenceNode private[yaml] ( + nodes: Seq[Node], + tag: Tag, + pos: Option[Range] = None + ) extends Node object SequenceNode: - def apply(nodes: Node*): SequenceNode = new SequenceNode(nodes, None) + def apply(nodes: Node*): SequenceNode = new SequenceNode(nodes, Tag.seq, None) def apply(first: YamlPrimitive, rest: YamlPrimitive*): SequenceNode = val nodes: List[YamlPrimitive] = (first :: rest.toList) - new SequenceNode(nodes.map(_.node), None) + new SequenceNode(nodes.map(_.node), Tag.seq, None) + def unapply(node: SequenceNode): Option[(Seq[Node], Tag)] = Some((node.nodes, node.tag)) + end SequenceNode - final case class MappingNode( - mappings: Seq[KeyValueNode], + final case class MappingNode private[yaml] ( + mappings: Map[Node, Node], + tag: Tag, pos: Option[Range] = None ) extends Node - object MappingNode: - def apply(nodes: KeyValueNode*): MappingNode = MappingNode(nodes, None) + def apply(mappings: Map[Node, Node]): MappingNode = MappingNode(mappings, Tag.map, None) + def apply(mappings: (Node, Node)*): MappingNode = MappingNode(mappings.toMap, Tag.map, None) def apply( first: (YamlPrimitive, YamlPrimitive), rest: (YamlPrimitive, YamlPrimitive)* ): MappingNode = - val nodes = (first :: rest.toList) - val kvn = nodes.map((k, v) => KeyValueNode(k.node, v.node)) - new MappingNode(kvn, None) + val primitives = (first :: rest.toList) + val mappings = primitives.map((k, v) => (k.node -> v.node)).toMap + new MappingNode(mappings, Tag.map, None) + def unapply(node: MappingNode): Option[(Map[Node, Node], Tag)] = Some((node.mappings, node.tag)) + end MappingNode +end Node - final case class KeyValueNode( - key: Node, - value: Node, - pos: Option[Range] = None - ) extends Node +private object TagResolver { + val nullPattern = "null|Null|NULL|~".r + val boolean = "true|True|TRUE|false|False|FALSE".r + val int10 = "[-+]?[0-9]+".r + val int8 = "0o[0-7]+".r + val int16 = "0x[0-9a-fA-F]+".r + val float = "[-+]?(\\.[0-9]+|[0-9]+(\\.[0-9]*)?)([eE][-+]?[0-9]+)?".r + val minusInfinity = "-(\\.inf|\\.Inf|\\.INF)".r + val plusInfinity = "\\+?(\\.inf|\\.Inf|\\.INF)".r -end Node + def resolveTag(value: String) = + value match + case null => 1 + case nullPattern(_*) => 2 + case boolean(_*) => 3 + case int10(_*) => 4 + case int8(_*) => 5 + case int16(_*) => 6 + case float(_*) => 7 + case minusInfinity(_*) => 8 + case plusInfinity(_*) => 9 +} diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala new file mode 100644 index 000000000..40c7845f1 --- /dev/null +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala @@ -0,0 +1,42 @@ +package org.virtuslab.yaml + +import scala.reflect.ClassTag + +final case class Tag(value: String) + +object Tag: + def apply[T](implicit classTag: ClassTag[T]): Tag = Tag(s"!${classTag.runtimeClass.getName}") + + private val default = "tag:yaml.org,2002:" + val nullTag: Tag = Tag(s"${default}null") + val boolean: Tag = Tag(s"${default}bool") + val int: Tag = Tag(s"${default}int") + val float: Tag = Tag(s"${default}float") + val str: Tag = Tag(s"${default}str") + val seq: Tag = Tag(s"${default}seq") + val map: Tag = Tag(s"${default}map") + + private val nullPattern = "null|Null|NULL|~".r + private val booleanPattern = "true|True|TRUE|false|False|FALSE".r + private val int10Pattern = "[-+]?[0-9]+".r + private val int8Pattern = "0o[0-7]+".r + private val int16Pattern = "0x[0-9a-fA-F]+".r + private val floatPattern = "[-+]?(\\.[0-9]+|[0-9]+(\\.[0-9]*)?)([eE][-+]?[0-9]+)?".r + private val minusInfinity = "-(\\.inf|\\.Inf|\\.INF)".r + private val plusInfinity = "\\+?(\\.inf|\\.Inf|\\.INF)".r + + def resolveTag(value: String): Tag = { + value match { + case null => nullTag + case nullPattern(_*) => nullTag + case booleanPattern(_*) => boolean + case int10Pattern(_*) => int + case int8Pattern(_*) => int + case int16Pattern(_*) => int + case floatPattern(_*) => float + case minusInfinity(_*) => float + case plusInfinity(_*) => float + case _ => str + } + } +end Tag diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala index d905c6c80..95e3470cc 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala @@ -81,7 +81,7 @@ object YamlDecoder: constructed.partitionMap(identity) match case (Nil, rights) => Right(rights) - case (lefts, _) => Left(lefts.head) + case (lefts) => ??? //Left(lefts.head) given [T](using c: YamlDecoder[T]): YamlDecoder[List[T]] = YamlDecoder { case SequenceNode(nodes, _) => @@ -104,8 +104,8 @@ object YamlDecoder: ): YamlDecoder[Map[K, V]] = YamlDecoder { case MappingNode(mappings, _) => val decoded: Seq[ Either[ConstructError, (K, V)] - ] = mappings - .map { case KeyValueNode(key, value, _) => + ] = mappings.toSeq + .map { (key, value) => (keyDecoder.construct(key) -> valueDecoder.construct(value)) } .map { case (key, value) => @@ -116,8 +116,8 @@ object YamlDecoder: } decoded.partitionMap(identity) match - case (Nil, rights) => Right(rights.toMap) - case (lefts, _) => Left(lefts.head) + case (lefts, _) if lefts.nonEmpty => Left(lefts.head) + case (_, rights) => Right(rights.toMap) } given YamlDecoder[String] = YamlDecoder { case ScalarNode(value, _) => @@ -129,18 +129,16 @@ object YamlDecoder: case s: Mirror.SumOf[T] => sumOf(s) private def extractKeyValues( - mappings: Seq[KeyValueNode] + mappings: Map[Node, Node] ): Either[ConstructError, Map[String, Node]] = { - val (error, valuesSeq) = mappings - .map { mapping => - val key = mapping.key match { - case ScalarNode(k, _) => Right(k) + val keyValueMap = mappings + .map { (k, v) => + k match { + case ScalarNode(scalarKey, _) => Right((scalarKey, v)) case _ => Left(ConstructError(s"Parameter of a class must be a scalar value")) } - val value = mapping.value - key.map(k => k -> value) } - .partitionMap(identity) + val (error, valuesSeq) = keyValueMap.partitionMap(identity) if (error.nonEmpty) Left(error.head) else Right(valuesSeq.toMap) @@ -170,9 +168,10 @@ object YamlDecoder: node match case Node.MappingNode(mappings, _) => for { - valuesMap <- extractKeyValues(mappings) - construcedValues <- constructValues(elemLabels, instances, valuesMap, p) - } yield (construcedValues) + valuesMap <- extractKeyValues(mappings) + + constructedValues <- constructValues(elemLabels, instances, valuesMap, p) + } yield (constructedValues) case _ => Left(ConstructError(s"Expected MappingNode, got ${node.getClass.getSimpleName}")) } diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala index 6766f9956..c5f7c1904 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala @@ -21,21 +21,19 @@ object YamlEncoder: given YamlEncoder[String] = v => Node.ScalarNode(v) given [T](using encoder: YamlEncoder[T]): YamlEncoder[Set[T]] = (nodes) => - Node.SequenceNode(nodes.map(encoder.asNode(_)).toSeq) + Node.SequenceNode(nodes.map(encoder.asNode(_)).toSeq, Tag("")) given [T](using encoder: YamlEncoder[T]): YamlEncoder[Seq[T]] = (nodes) => - Node.SequenceNode(nodes.map(encoder.asNode(_))) + Node.SequenceNode(nodes.map(encoder.asNode(_)), Tag("")) given [T](using encoder: YamlEncoder[T]): YamlEncoder[List[T]] = (nodes) => - Node.SequenceNode(nodes.map(encoder.asNode(_))) + Node.SequenceNode(nodes.map(encoder.asNode(_)), Tag("")) // todo support arbitrary node as key in KeyValueNode - given [V](using valueCodec: YamlEncoder[V]): YamlEncoder[Map[String, V]] = + given [K, V](using keyCodec: YamlEncoder[K], valueCodec: YamlEncoder[V]): YamlEncoder[Map[K, V]] = (nodes) => - val pairs = nodes.toList.map((key, value) => - Node.KeyValueNode(Node.ScalarNode(key), valueCodec.asNode(value)) - ) - Node.MappingNode(pairs) + val mappings = nodes.map((key, value) => (keyCodec.asNode(key) -> valueCodec.asNode(value))) + Node.MappingNode(mappings) inline def derived[T](using m: Mirror.Of[T]): YamlEncoder[T] = inline m match case p: Mirror.ProductOf[T] => deriveProduct(p) @@ -48,11 +46,15 @@ object YamlEncoder: override def asNode(obj: T): Node = val products = obj.asInstanceOf[Product].productIterator val nodes = - elemLabels.zip(products).zip(yamlEncoders).map { case ((label, element), encoder) => - val key = Node.ScalarNode(label) - val value = encoder.asInstanceOf[YamlEncoder[Any]].asNode(element) - Node.KeyValueNode(key, value) - } + elemLabels + .zip(products) + .zip(yamlEncoders) + .map { case ((label, element), encoder) => + val key: Node = Node.ScalarNode(label) + val value = encoder.asInstanceOf[YamlEncoder[Any]].asNode(element) + (key, value) + } + .toMap Node.MappingNode(nodes) } diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala index 0e2fcf399..119d4524a 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/dump/serialize/SerializerImpl.scala @@ -13,18 +13,14 @@ object SerializerImpl extends Serializer: case scalar: Node.ScalarNode => convertScalarNode(scalar) case mapping: Node.MappingNode => convertMappingNode(mapping) case sequence: Node.SequenceNode => convertSequenceNode(sequence) - case kv: Node.KeyValueNode => convertKeyValueNode(kv) private def convertMappingNode(node: Node.MappingNode): Seq[EventKind] = - val events = node.mappings.map(convertNode(_)).flatten + val events = node.mappings.toSeq.flatMap((k, v) => Seq(convertNode(k), convertNode(v))).flatten Seq(MappingStart()) ++ events ++ Seq(MappingEnd) private def convertSequenceNode(node: Node.SequenceNode): Seq[EventKind] = val events = node.nodes.map(convertNode(_)).flatten Seq(SequenceStart()) ++ events ++ Seq(SequenceEnd) - private def convertKeyValueNode(node: Node.KeyValueNode): Seq[EventKind] = - convertNode(node.key) ++ convertNode(node.value) - private def convertScalarNode(node: Node.ScalarNode): Seq[EventKind] = Seq(Scalar(node.value)) diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala index b11aecc26..4fa141438 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala @@ -6,6 +6,7 @@ import org.virtuslab.yaml.ComposerError import org.virtuslab.yaml.Node import org.virtuslab.yaml.Position import org.virtuslab.yaml.Range +import org.virtuslab.yaml.Tag import org.virtuslab.yaml.YamlError import org.virtuslab.yaml.internal.load.parse.Event import org.virtuslab.yaml.internal.load.parse.EventKind @@ -37,7 +38,8 @@ object ComposerImpl extends Composer: case _: EventKind.SequenceStart => composeSequenceNode(tail) case _: EventKind.MappingStart | _: EventKind.FlowMappingStart => composeMappingNode(tail) case s: EventKind.Scalar => - Right(Result(Node.ScalarNode(s.value, head.pos), tail)) + val tag: Tag = s.metadata.tag.getOrElse(Tag.resolveTag(s.value)) + Right(Result(Node.ScalarNode(s.value, tag, head.pos), tail)) // todo #88 case _: EventKind.Alias => Left(ComposerError(s"Aliases aren't currently supported")) case event => Left(ComposerError(s"Expected YAML node, but found: $event")) @@ -60,7 +62,7 @@ object ComposerImpl extends Composer: case Left(err) => Left(err) parseChildren(events, Nil).map { case (Result(nodes, rest), pos) => - Result(Node.SequenceNode(nodes, pos), rest) + Result(new Node.SequenceNode(nodes, Tag.seq, pos), rest) } } @@ -68,9 +70,9 @@ object ComposerImpl extends Composer: @tailrec def parseMappings( events: List[Event], - mappings: List[Node.KeyValueNode], + mappings: List[(Node, Node)], firstChildPos: Option[Range] = None - ): ComposeResultWithPos[List[Node.KeyValueNode]] = { + ): ComposeResultWithPos[List[(Node, Node)]] = { events match case Nil => Left(ComposerError("Not found MappingEnd event for mapping")) case Event(EventKind.MappingEnd | EventKind.FlowMappingEnd, _) :: tail => @@ -86,14 +88,16 @@ object ComposerImpl extends Composer: for key <- composeNode(events) v <- composeNode(key.remaining) - yield Result(Node.KeyValueNode(key.node, v.node, key.node.pos), v.remaining) - + yield Result((key.node, v.node), v.remaining) mapping match - case Right(node, rest) => parseMappings(rest, mappings :+ node, node.pos) - case Left(err) => Left(err) + case Right(node @ (key, value), rest) => parseMappings(rest, mappings :+ node, key.pos) + case Left(err) => Left(err) } parseMappings(events, Nil).map { case (Result(nodes, rest), pos) => - Result(Node.MappingNode(nodes, pos), rest) + Result( + new Node.MappingNode(nodes.toMap, Tag.map, pos), + rest + ) } } diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/Event.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/Event.scala index 852bfaba9..6902b2c4f 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/Event.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/Event.scala @@ -2,6 +2,7 @@ package org.virtuslab.yaml.internal.load.parse import org.virtuslab.yaml.Node import org.virtuslab.yaml.Range +import org.virtuslab.yaml.Tag import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle /** @@ -67,7 +68,3 @@ object NodeEventMetadata: opaque type Anchor = String object Anchor: def apply(anchor: String): Anchor = anchor - -opaque type Tag = String -object Tag: - def apply(tag: String): Tag = tag diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala index b12b996dd..74f0b07c0 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala @@ -8,6 +8,7 @@ import scala.util.Try import org.virtuslab.yaml.ParseError import org.virtuslab.yaml.Range +import org.virtuslab.yaml.Tag import org.virtuslab.yaml.YamlError import org.virtuslab.yaml.internal.load.TagHandle import org.virtuslab.yaml.internal.load.TagValue diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/ComposerSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/ComposerSuite.scala index e77f770d3..36f7a706a 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/ComposerSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/ComposerSuite.scala @@ -34,7 +34,7 @@ class ComposerSuite extends munit.FunSuite: assertEquals(ComposerImpl.fromEvents(events), expected) } - test("mapping of scalars") { + test("mapping of scalars".only) { val events = List( StreamStart, DocumentStart(), @@ -52,9 +52,9 @@ class ComposerSuite extends munit.FunSuite: val expected = Right( MappingNode( - KeyValueNode(ScalarNode("hr"), ScalarNode("65")), - KeyValueNode(ScalarNode("avg"), ScalarNode("0.278")), - KeyValueNode(ScalarNode("rbi"), ScalarNode("147")) + ScalarNode("hr") -> ScalarNode("65"), + ScalarNode("avg") -> ScalarNode("0.278"), + ScalarNode("rbi") -> ScalarNode("147") ) ) @@ -85,23 +85,19 @@ class ComposerSuite extends munit.FunSuite: val expected = Right( MappingNode( - List( - KeyValueNode( - ScalarNode("american"), + Map( + ScalarNode("american") -> SequenceNode( ScalarNode("Boston Red Sox"), ScalarNode("Detroit Tigers"), ScalarNode("New York Yankees") - ) - ), - KeyValueNode( - ScalarNode("national"), + ), + ScalarNode("national") -> SequenceNode( ScalarNode("New York Mets"), ScalarNode("Chicago Cubs"), ScalarNode("Atlanta Braves") ) - ) ) ) ) @@ -138,14 +134,14 @@ class ComposerSuite extends munit.FunSuite: val expected = Right( SequenceNode( MappingNode( - KeyValueNode(ScalarNode("name"), ScalarNode("Mark McGwire")), - KeyValueNode(ScalarNode("hr"), ScalarNode("65")), - KeyValueNode(ScalarNode("avg"), ScalarNode("0.278")) + ScalarNode("name") -> ScalarNode("Mark McGwire"), + ScalarNode("hr") -> ScalarNode("65"), + ScalarNode("avg") -> ScalarNode("0.278") ), MappingNode( - KeyValueNode(ScalarNode("name"), ScalarNode("Sammy Sosa")), - KeyValueNode(ScalarNode("hr"), ScalarNode("63")), - KeyValueNode(ScalarNode("avg"), ScalarNode("0.288")) + ScalarNode("name") -> ScalarNode("Sammy Sosa"), + ScalarNode("hr") -> ScalarNode("63"), + ScalarNode("avg") -> ScalarNode("0.288") ) ) ) diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala index 6cdb01617..e04b21527 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala @@ -15,19 +15,19 @@ class ConstructSuite extends munit.FunSuite: test("derive construct for case class") { val node = MappingNode( - KeyValueNode(ScalarNode("hr"), ScalarNode("65")), - KeyValueNode(ScalarNode("avg"), ScalarNode("0.278")), - KeyValueNode(ScalarNode("rbi"), ScalarNode("147")) + ScalarNode("hr") -> ScalarNode("65"), + ScalarNode("avg") -> ScalarNode("0.278"), + ScalarNode("rbi") -> ScalarNode("147") ) val expected = Right(Stats(65, 0.278, 147)) assertEquals(node.as[Stats], expected) } test("derive construct for sealed trait") { - val foo = MappingNode(KeyValueNode(ScalarNode("value"), ScalarNode("65"))) + val foo = MappingNode(ScalarNode("value") -> ScalarNode("65")) assertEquals(foo.as[SomeEnum], Right(SomeEnum.Foo(65))) - val bar = MappingNode(KeyValueNode(ScalarNode("price"), ScalarNode("65.997"))) + val bar = MappingNode(ScalarNode("price") -> ScalarNode("65.997")) assertEquals(bar.as[SomeEnum], Right(SomeEnum.Bar(65.997))) } diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/TagSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/TagSuite.scala index f1eb24764..46e3a704a 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/TagSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/TagSuite.scala @@ -1,11 +1,11 @@ package org.virtuslab.yaml package parser +import org.virtuslab.yaml.Tag import org.virtuslab.yaml.internal.load.parse.Anchor import org.virtuslab.yaml.internal.load.parse.EventKind import org.virtuslab.yaml.internal.load.parse.EventKind.* import org.virtuslab.yaml.internal.load.parse.NodeEventMetadata -import org.virtuslab.yaml.internal.load.parse.Tag import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle class TagSuite extends BaseYamlSuite: diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/tokenizer/TagSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/tokenizer/TagSuite.scala index e7a525c51..30a21d94d 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/tokenizer/TagSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/tokenizer/TagSuite.scala @@ -1,7 +1,6 @@ package org.virtuslab.yaml package tokenizer -import org.virtuslab.yaml.* import org.virtuslab.yaml.internal.load.TagHandle import org.virtuslab.yaml.internal.load.TagPrefix import org.virtuslab.yaml.internal.load.TagValue From 40925590c4a5635a902499b2f4c9130ced75ff69 Mon Sep 17 00:00:00 2001 From: Kamil Podsiadlo Date: Sat, 13 Nov 2021 08:29:05 +0100 Subject: [PATCH 2/3] Try to automatically detect types and use them when decoding yaml --- .../org/virtuslab/yaml/LoadSettings.scala | 5 + .../main/scala/org/virtuslab/yaml/Node.scala | 27 +--- .../main/scala/org/virtuslab/yaml/Tag.scala | 2 + .../main/scala/org/virtuslab/yaml/Yaml.scala | 7 +- .../scala/org/virtuslab/yaml/YamlCodec.scala | 6 +- .../org/virtuslab/yaml/YamlDecoder.scala | 117 ++++++++++++------ .../org/virtuslab/yaml/YamlEncoder.scala | 6 +- .../scala/org/virtuslab/yaml/YamlError.scala | 37 +++++- .../org/virtuslab/yaml/ConstructSuite.scala | 16 +++ .../yaml/decoder/DecoderErrorsSuite.scala | 2 +- ...eDecoderSuite.scala => DecoderSuite.scala} | 49 +++++++- 11 files changed, 197 insertions(+), 77 deletions(-) create mode 100644 yaml/shared/src/main/scala/org/virtuslab/yaml/LoadSettings.scala rename yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/{PrimitiveDecoderSuite.scala => DecoderSuite.scala} (80%) diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/LoadSettings.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/LoadSettings.scala new file mode 100644 index 000000000..46d1b2baa --- /dev/null +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/LoadSettings.scala @@ -0,0 +1,5 @@ +package org.virtuslab.yaml + +case class LoadSettings(constructors: Map[Tag, YamlDecoder[Any]]) +object LoadSettings: + val empty = LoadSettings(Map.empty) diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala index 9aa467ed9..966e4251c 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala @@ -10,6 +10,10 @@ import org.virtuslab.yaml.syntax.YamlPrimitive sealed trait Node: private[yaml] def pos: Option[Range] def tag: Tag + def as[T](using + c: YamlDecoder[T], + settings: LoadSettings = LoadSettings.empty + ): Either[YamlError, T] = c.construct(this) object Node: final case class ScalarNode private[yaml] (value: String, tag: Tag, pos: Option[Range] = None) @@ -51,26 +55,3 @@ object Node: def unapply(node: MappingNode): Option[(Map[Node, Node], Tag)] = Some((node.mappings, node.tag)) end MappingNode end Node - -private object TagResolver { - val nullPattern = "null|Null|NULL|~".r - val boolean = "true|True|TRUE|false|False|FALSE".r - val int10 = "[-+]?[0-9]+".r - val int8 = "0o[0-7]+".r - val int16 = "0x[0-9a-fA-F]+".r - val float = "[-+]?(\\.[0-9]+|[0-9]+(\\.[0-9]*)?)([eE][-+]?[0-9]+)?".r - val minusInfinity = "-(\\.inf|\\.Inf|\\.INF)".r - val plusInfinity = "\\+?(\\.inf|\\.Inf|\\.INF)".r - - def resolveTag(value: String) = - value match - case null => 1 - case nullPattern(_*) => 2 - case boolean(_*) => 3 - case int10(_*) => 4 - case int8(_*) => 5 - case int16(_*) => 6 - case float(_*) => 7 - case minusInfinity(_*) => 8 - case plusInfinity(_*) => 9 -} diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala index 40c7845f1..d82863e65 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala @@ -16,6 +16,8 @@ object Tag: val seq: Tag = Tag(s"${default}seq") val map: Tag = Tag(s"${default}map") + val primitives = Set(nullTag, boolean, int, float, str) + private val nullPattern = "null|Null|NULL|~".r private val booleanPattern = "true|True|TRUE|false|False|FALSE".r private val int10Pattern = "[-+]?[0-9]+".r diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/Yaml.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/Yaml.scala index 63b5e0c4a..d5b34614f 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/Yaml.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/Yaml.scala @@ -13,8 +13,6 @@ inline def deriveYamlEncoder[T](using m: Mirror.Of[T]): YamlEncoder[T] = YamlEnc inline def deriveYamlDecoder[T](using m: Mirror.Of[T]): YamlDecoder[T] = YamlDecoder.derived[T] inline def deriveYamlCodec[T](using m: Mirror.Of[T]): YamlCodec[T] = YamlCodec.derived[T] -extension (node: Node) def as[T](using c: YamlDecoder[T]): Either[YamlError, T] = c.construct(node) - extension (str: String) /** * Parse YAML from the given [[String]], returning either [[YamlError]] or [[T]]. @@ -24,7 +22,10 @@ extension (str: String) * - then [[Composer]] produces a representation graph from events * - finally [[YamlDecoder]] (construct phase from the YAML spec) constructs data type [[T]] from the YAML representation. */ - def as[T](using c: YamlDecoder[T]): Either[YamlError, T] = + def as[T](using + c: YamlDecoder[T], + settings: LoadSettings = LoadSettings.empty + ): Either[YamlError, T] = for events <- { val parser = ParserImpl(Scanner(str)) diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlCodec.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlCodec.scala index 9801bf3ea..3b2ad300c 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlCodec.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlCodec.scala @@ -16,7 +16,9 @@ object YamlCodec: val decoder = YamlDecoder.derived[T] val encoder = YamlEncoder.derived[T] - def construct(node: Node): Either[ConstructError, T] = decoder.construct(node) - def asNode(obj: T): Node = encoder.asNode(obj) + def construct(node: Node)(using + settings: LoadSettings = LoadSettings.empty + ): Either[ConstructError, T] = decoder.construct(node) + def asNode(obj: T): Node = encoder.asNode(obj) end YamlCodec diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala index 95e3470cc..b5d882ec3 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala @@ -12,64 +12,96 @@ import org.virtuslab.yaml.Node.* * A type class that provides a conversion from a [[Node]] into given type [[T]] */ trait YamlDecoder[T]: - def construct(node: Node): Either[ConstructError, T] + def construct(node: Node)(using + settings: LoadSettings = LoadSettings.empty + ): Either[ConstructError, T] object YamlDecoder: def apply[T]( - pf: PartialFunction[Node, Either[ConstructError | Throwable, T]] - )(implicit tag: ClassTag[T]): YamlDecoder[T] = + pf: PartialFunction[Node, Either[ConstructError, T]] + )(using classTag: ClassTag[T]): YamlDecoder[T] = new YamlDecoder[T] { - override def construct(node: Node): Either[ConstructError, T] = - if pf.isDefinedAt(node) then - pf(node) match { - case Left(e: Throwable) => - val msg = node.pos match { - case Some(pos) => - s"""|${e.getMessage} - |at ${pos.start.line}:${pos.start.column}, expected $tag - |${pos.errorMsg} """.stripMargin - case None => - "Cannot decode: ${e.getMessage}" - } - Left(ConstructError(msg)) - case Left(e: ConstructError) => Left(e) - case Right(v) => Right(v) - } - else Left(ConstructError(s"Could't create Construct instance for $node")) + override def construct( + node: Node + )(using settings: LoadSettings = LoadSettings.empty): Either[ConstructError, T] = + if pf.isDefinedAt(node) then pf(node) + else Left(ConstructError(s"Could't construct ${classTag.runtimeClass.getName} from $node")) } - given YamlDecoder[Int] = YamlDecoder { case ScalarNode(value, _) => - Try(value.toInt).toEither + private inline def cannotParse(value: Any, tpe: String, node: Node) = ConstructError.from( + s"Cannot parse $value as $tpe", + node, + tpe + ) + + given YamlDecoder[Int] = YamlDecoder { case s @ ScalarNode(value, _) => + value.toIntOption.toRight(cannotParse(value, "Int", s)) } - given YamlDecoder[Long] = YamlDecoder { case ScalarNode(value, _) => - Try(value.toLong).toEither + given YamlDecoder[Long] = YamlDecoder { case s @ ScalarNode(value, _) => + value.toLongOption.toRight(cannotParse(value, "Long", s)) } - given YamlDecoder[Double] = YamlDecoder { case ScalarNode(value, _) => - Try(value.toDouble).toEither + given YamlDecoder[Double] = YamlDecoder { case s @ ScalarNode(value, _) => + value.toDoubleOption.toRight(cannotParse(value, "Double", s)) } - given YamlDecoder[Float] = YamlDecoder { case ScalarNode(value, _) => - Try(value.toFloat).toEither + given YamlDecoder[Float] = YamlDecoder { case s @ ScalarNode(value, _) => + value.toFloatOption.toRight(cannotParse(value, "Float", s)) } - given YamlDecoder[Short] = YamlDecoder { case ScalarNode(value, _) => - Try(value.toShort).toEither + given YamlDecoder[Short] = YamlDecoder { case s @ ScalarNode(value, _) => + value.toShortOption.toRight(cannotParse(value, "Short", s)) } - given YamlDecoder[Byte] = YamlDecoder { case ScalarNode(value, _) => - Try(value.toByte).toEither + given YamlDecoder[Byte] = YamlDecoder { case s @ ScalarNode(value, _) => + value.toByteOption.toRight(cannotParse(value, "Byte", s)) } - given YamlDecoder[Boolean] = YamlDecoder { case ScalarNode(value, _) => - Try(value.toBoolean).toEither + given YamlDecoder[Boolean] = YamlDecoder { case s @ ScalarNode(value, _) => + value.toBooleanOption.toRight(cannotParse(value, "Boolean", s)) + } + + given YamlDecoder[Any] = new YamlDecoder { + def construct(node: Node)(using settings: LoadSettings = LoadSettings.empty) = { + node match { + case ScalarNode(value, tag) if Tag.primitives.contains(tag) => + tag match { + case Tag.nullTag => Right(None) + case Tag.boolean => value.toBooleanOption.toRight(cannotParse(value, "Boolean", node)) + case Tag.int => + if value.startsWith("0b") then + Try(Integer.parseInt(value.drop(2), 8)).toEither.swap + .map(t => ConstructError.from(t, "Int", node)) + .swap + else if value.startsWith("0x") then + Try(Integer.parseInt(value.drop(2), 8)).toEither.swap + .map(t => ConstructError.from(t, "Int", node)) + .swap + else value.toIntOption.toRight(cannotParse(value, "Int", node)) + case Tag.float => + value.toDoubleOption.toRight(cannotParse(value, "Double", node)) + case Tag.str => Right(value) + } + case MappingNode(mappings, Tag.map) => + val decoder = summon[YamlDecoder[Map[Any, Any]]] + decoder.construct(node) + case SequenceNode(seq, Tag.seq) => + val decoder = summon[YamlDecoder[Seq[Any]]] + decoder.construct(node) + case _ => + settings.constructors.get(node.tag) match { + case Some(decoder) => decoder.construct(node) + case None => Left(ConstructError(s"Could't create construct instance for ${node.tag}")) + } + } + } } given [T](using c: YamlDecoder[T]): YamlDecoder[Option[T]] = YamlDecoder { - case ScalarNode(value, _) => + case ScalarNode(value, tag) => value match - case "null" | "" => Right(None) + case _ if tag == Tag.nullTag => Right(None) case _ => c.construct(ScalarNode(value)).map(Option(_)) } @@ -81,7 +113,7 @@ object YamlDecoder: constructed.partitionMap(identity) match case (Nil, rights) => Right(rights) - case (lefts) => ??? //Left(lefts.head) + case (lefts, _) => Left(lefts.head) given [T](using c: YamlDecoder[T]): YamlDecoder[List[T]] = YamlDecoder { case SequenceNode(nodes, _) => @@ -164,12 +196,13 @@ object YamlDecoder: val instances = summonAll[p.MirroredElemTypes] val elemLabels = getElemLabels[p.MirroredElemLabels] new YamlDecoder[T] { - override def construct(node: Node): Either[ConstructError, T] = + override def construct(node: Node)(using + constructor: LoadSettings = LoadSettings.empty + ): Either[ConstructError, T] = node match case Node.MappingNode(mappings, _) => for { - valuesMap <- extractKeyValues(mappings) - + valuesMap <- extractKeyValues(mappings) constructedValues <- constructValues(elemLabels, instances, valuesMap, p) } yield (constructedValues) case _ => @@ -179,7 +212,9 @@ object YamlDecoder: private inline def sumOf[T](s: Mirror.SumOf[T]) = val instances = summonSumOf[s.MirroredElemTypes].asInstanceOf[List[YamlDecoder[T]]] new YamlDecoder[T]: - override def construct(node: Node): Either[ConstructError, T] = LazyList + override def construct( + node: Node + )(using constructor: LoadSettings = LoadSettings.empty): Either[ConstructError, T] = LazyList .from(instances) .map(c => c.construct(node)) .collectFirst { case r @ Right(_) => r } diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala index c5f7c1904..1b44828fc 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlEncoder.scala @@ -21,13 +21,13 @@ object YamlEncoder: given YamlEncoder[String] = v => Node.ScalarNode(v) given [T](using encoder: YamlEncoder[T]): YamlEncoder[Set[T]] = (nodes) => - Node.SequenceNode(nodes.map(encoder.asNode(_)).toSeq, Tag("")) + Node.SequenceNode(nodes.map(encoder.asNode(_)).toSeq, Tag.seq) given [T](using encoder: YamlEncoder[T]): YamlEncoder[Seq[T]] = (nodes) => - Node.SequenceNode(nodes.map(encoder.asNode(_)), Tag("")) + Node.SequenceNode(nodes.map(encoder.asNode(_)), Tag.seq) given [T](using encoder: YamlEncoder[T]): YamlEncoder[List[T]] = (nodes) => - Node.SequenceNode(nodes.map(encoder.asNode(_)), Tag("")) + Node.SequenceNode(nodes.map(encoder.asNode(_)), Tag.seq) // todo support arbitrary node as key in KeyValueNode given [K, V](using keyCodec: YamlEncoder[K], valueCodec: YamlEncoder[V]): YamlEncoder[Map[K, V]] = diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlError.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlError.scala index 73d4cf663..c6bb114e4 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlError.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlError.scala @@ -1,5 +1,6 @@ package org.virtuslab.yaml +import scala.reflect.ClassTag import scala.util.control.NoStackTrace import org.virtuslab.yaml.internal.load.reader.token.Token @@ -20,12 +21,42 @@ object ParseError: ) def from(expected: TokenKind, got: Token): ParseError = ParseError.from(expected.toString, got) -final case class ComposerError(msg: String) extends YamlError +final case class ComposerError(msg: String) extends YamlError + final case class ConstructError(msg: String) extends YamlError -final case class ScannerError(msg: String) extends Throwable with YamlError with NoStackTrace +object ConstructError: + private def from(errorMsg: String, node: Option[Node], expected: Option[String]): ConstructError = + val msg = node.flatMap(_.pos) match + case Some(range) => + s"""|$errorMsg + |at ${range.start.line}:${range.start.column},${expected.map(exp => s" expected $exp").getOrElse("")} + |${range.errorMsg} """.stripMargin + case None => + errorMsg + ConstructError(msg) + def from(errorMsg: String, node: Node, expected: String): ConstructError = + from(errorMsg, Some(node), Some(expected)) + def from(errorMsg: String, expected: String, node: Node): ConstructError = + from(errorMsg, Some(node), Some(expected)) + def from(errorMsg: String, node: Node): ConstructError = from(errorMsg, Some(node), None) + def from(errorMsg: String, expected: String): ConstructError = + from(errorMsg, None, Some(expected)) + def from(errorMsg: String): ConstructError = from(errorMsg, None, None) + + def from(t: Throwable, node: Node, expected: String): ConstructError = + from(t.getMessage, Some(node), Some(expected)) + def from(t: Throwable, expected: String, node: Node): ConstructError = + from(t.getMessage, Some(node), Some(expected)) + def from(t: Throwable, node: Node): ConstructError = from(t.getMessage, Some(node), None) + def from(t: Throwable, expected: String): ConstructError = + from(t.getMessage, None, Some(expected)) + def from(t: Throwable): ConstructError = from(t.getMessage, None, None) +end ConstructError + +final case class ScannerError(msg: String) extends Throwable with YamlError with NoStackTrace object ScannerError: def from(obtained: String, got: Token): ScannerError = ScannerError( s"""|Obtained - |$obtained but exptected got ${got.kind} + |$obtained but expected got ${got.kind} |${got.range.errorMsg}""".stripMargin ) diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala index e04b21527..504a2f235 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala @@ -45,3 +45,19 @@ class ConstructSuite extends munit.FunSuite: Left(ConstructError(s"Parameter of a class must be a scalar value")) assertEquals(node.as[DummyClass], expectedConstructError) } + + test("decode as Any".only) { + + val node = MappingNode( + SequenceNode(1, 2) -> ScalarNode("seq") + ) + + val expected = Map[Any, Any]( + Seq(1, 2) -> "seq" + ) + + println(SequenceNode(1, 2)) + println(node.as[Any]) + + assertEquals(node.as[Any], Right(expected)) + } diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderErrorsSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderErrorsSuite.scala index c4f9a31b1..7e7549103 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderErrorsSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderErrorsSuite.scala @@ -17,7 +17,7 @@ class DecoderErrorsSuite extends BaseDecoderErrorSuite: assertError( yaml.as[Person], - s"""|For input string: "xxx" + s"""|Cannot parse xxx as Int |at 1:5, expected Int |age: xxx | ^ diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/PrimitiveDecoderSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderSuite.scala similarity index 80% rename from yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/PrimitiveDecoderSuite.scala rename to yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderSuite.scala index c981b3f98..d7e1cf28e 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/PrimitiveDecoderSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderSuite.scala @@ -1,8 +1,9 @@ package org.virtuslab.yaml.decoder +import org.virtuslab.yaml.Node.ScalarNode import org.virtuslab.yaml.* -class PrimitiveDecoderSuite extends munit.FunSuite: +class DecoderSuite extends munit.FunSuite: test("numbers") { @@ -184,3 +185,49 @@ class PrimitiveDecoderSuite extends munit.FunSuite: assertEquals(yaml.as[Spec], Right(expectedSpec)) } + + test("decode into Map[Any, Any]") { + + val yaml = + s"""|123: 321 + |string: aezakmi + |true: false + |5.5: 55.55 + |""".stripMargin + + val expected = Map[Any, Any]( + 123 -> 321, + "string" -> "aezakmi", + true -> false, + 5.5 -> 55.55 + ) + + assertEquals(yaml.as[Map[Any, Any]], Right(expected)) + } + + test("decode using custom tag".only) { + case class Custom(x: Int, doubledX: Int) + + val yaml = + s"""|!Custom 5 + |""".stripMargin + + val expected = Custom(5, 10) + + val decoder = new YamlDecoder[Custom] { + override def construct(node: Node)(using + settings: LoadSettings = LoadSettings.empty + ): Either[ConstructError, Custom] = node match { + case ScalarNode(value, _) => + val int = value.toInt + Right(Custom(int, int * 2)) + case _ => ??? + } + }.asInstanceOf[YamlDecoder[Any]] + + given settings: LoadSettings = LoadSettings( + Map(Tag("!Custom") -> decoder) + ) + + assertEquals(yaml.as[Any], Right(expected)) + } From 7fa721b53b82eb24ef3046d54b84cae7fb69df72 Mon Sep 17 00:00:00 2001 From: Kamil Podsiadlo Date: Sat, 13 Nov 2021 13:49:39 +0100 Subject: [PATCH 3/3] Review fixes --- .../org/virtuslab/yaml/LoadSettings.scala | 2 +- .../main/scala/org/virtuslab/yaml/Node.scala | 3 +- .../main/scala/org/virtuslab/yaml/Tag.scala | 29 ++++++++++++------- .../org/virtuslab/yaml/YamlDecoder.scala | 26 ++++++++++++----- .../yaml/internal/load/compose/Composer.scala | 4 +-- .../yaml/internal/load/parse/Event.scala | 3 +- .../yaml/internal/load/parse/ParserImpl.scala | 14 ++++++--- .../org/virtuslab/yaml/ComposerSuite.scala | 2 +- .../org/virtuslab/yaml/ConstructSuite.scala | 5 +--- .../virtuslab/yaml/decoder/DecoderSuite.scala | 18 ++++-------- .../virtuslab/yaml/parser/AnchorSpec.scala | 2 +- .../virtuslab/yaml/parser/MappingSuite.scala | 9 +++--- .../org/virtuslab/yaml/parser/TagSuite.scala | 21 +++++++------- 13 files changed, 78 insertions(+), 60 deletions(-) diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/LoadSettings.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/LoadSettings.scala index 46d1b2baa..ca7ce1db6 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/LoadSettings.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/LoadSettings.scala @@ -1,5 +1,5 @@ package org.virtuslab.yaml -case class LoadSettings(constructors: Map[Tag, YamlDecoder[Any]]) +case class LoadSettings(constructors: Map[Tag, YamlDecoder[?]]) object LoadSettings: val empty = LoadSettings(Map.empty) diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala index 966e4251c..ab0a20eb0 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/Node.scala @@ -13,7 +13,8 @@ sealed trait Node: def as[T](using c: YamlDecoder[T], settings: LoadSettings = LoadSettings.empty - ): Either[YamlError, T] = c.construct(this) + ): Either[YamlError, T] = + c.construct(this) object Node: final case class ScalarNode private[yaml] (value: String, tag: Tag, pos: Option[Range] = None) diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala index d82863e65..e9c0a98f1 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/Tag.scala @@ -2,21 +2,28 @@ package org.virtuslab.yaml import scala.reflect.ClassTag -final case class Tag(value: String) +sealed trait Tag: + def value: String + +final case class CoreSchemaTag(value: String) extends Tag +final case class CustomTag(value: String) extends Tag object Tag: - def apply[T](implicit classTag: ClassTag[T]): Tag = Tag(s"!${classTag.runtimeClass.getName}") + def apply[T](implicit classTag: ClassTag[T]): Tag = CustomTag( + s"!${classTag.runtimeClass.getName}" + ) private val default = "tag:yaml.org,2002:" - val nullTag: Tag = Tag(s"${default}null") - val boolean: Tag = Tag(s"${default}bool") - val int: Tag = Tag(s"${default}int") - val float: Tag = Tag(s"${default}float") - val str: Tag = Tag(s"${default}str") - val seq: Tag = Tag(s"${default}seq") - val map: Tag = Tag(s"${default}map") - - val primitives = Set(nullTag, boolean, int, float, str) + val nullTag: Tag = CoreSchemaTag(s"${default}null") + val boolean: Tag = CoreSchemaTag(s"${default}bool") + val int: Tag = CoreSchemaTag(s"${default}int") + val float: Tag = CoreSchemaTag(s"${default}float") + val str: Tag = CoreSchemaTag(s"${default}str") + val seq: Tag = CoreSchemaTag(s"${default}seq") + val map: Tag = CoreSchemaTag(s"${default}map") + + val corePrimitives = Set(nullTag, boolean, int, float, str) + val coreSchemaValues = (corePrimitives ++ Set(seq, map)).map(_.value) private val nullPattern = "null|Null|NULL|~".r private val booleanPattern = "true|True|TRUE|false|False|FALSE".r diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala index b5d882ec3..26a59ba15 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/YamlDecoder.scala @@ -25,7 +25,12 @@ object YamlDecoder: node: Node )(using settings: LoadSettings = LoadSettings.empty): Either[ConstructError, T] = if pf.isDefinedAt(node) then pf(node) - else Left(ConstructError(s"Could't construct ${classTag.runtimeClass.getName} from $node")) + else + Left( + ConstructError(s"""|Could't construct ${classTag.runtimeClass.getName} from ${node.tag} + |${node.pos.map(_.errorMsg).getOrElse("")} + |""".stripMargin) + ) } private inline def cannotParse(value: Any, tpe: String, node: Node) = ConstructError.from( @@ -65,19 +70,17 @@ object YamlDecoder: given YamlDecoder[Any] = new YamlDecoder { def construct(node: Node)(using settings: LoadSettings = LoadSettings.empty) = { node match { - case ScalarNode(value, tag) if Tag.primitives.contains(tag) => + case ScalarNode(value, tag: CoreSchemaTag) if Tag.corePrimitives.contains(tag) => tag match { case Tag.nullTag => Right(None) case Tag.boolean => value.toBooleanOption.toRight(cannotParse(value, "Boolean", node)) case Tag.int => if value.startsWith("0b") then - Try(Integer.parseInt(value.drop(2), 8)).toEither.swap + Try(Integer.parseInt(value.drop(2), 8)).toEither.left .map(t => ConstructError.from(t, "Int", node)) - .swap else if value.startsWith("0x") then - Try(Integer.parseInt(value.drop(2), 8)).toEither.swap + Try(Integer.parseInt(value.drop(2), 8)).toEither.left .map(t => ConstructError.from(t, "Int", node)) - .swap else value.toIntOption.toRight(cannotParse(value, "Int", node)) case Tag.float => value.toDoubleOption.toRight(cannotParse(value, "Double", node)) @@ -92,7 +95,16 @@ object YamlDecoder: case _ => settings.constructors.get(node.tag) match { case Some(decoder) => decoder.construct(node) - case None => Left(ConstructError(s"Could't create construct instance for ${node.tag}")) + case None => + Left( + ConstructError( + s"""|Could't construct runtime instance of ${node.tag} + |${node.pos.map(_.errorMsg).getOrElse("")} + |If you're using custom datatype consider using yaml.as[MyType] instead of Any + |Or define LoadSettings where you'll specify how to construct ${node.tag} + |""".stripMargin + ) + ) } } } diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala index 4fa141438..c47455938 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/compose/Composer.scala @@ -62,7 +62,7 @@ object ComposerImpl extends Composer: case Left(err) => Left(err) parseChildren(events, Nil).map { case (Result(nodes, rest), pos) => - Result(new Node.SequenceNode(nodes, Tag.seq, pos), rest) + Result(Node.SequenceNode(nodes, Tag.seq, pos), rest) } } @@ -96,7 +96,7 @@ object ComposerImpl extends Composer: parseMappings(events, Nil).map { case (Result(nodes, rest), pos) => Result( - new Node.MappingNode(nodes.toMap, Tag.map, pos), + Node.MappingNode(nodes.toMap, Tag.map, pos), rest ) } diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/Event.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/Event.scala index 6902b2c4f..e2964d125 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/Event.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/Event.scala @@ -62,7 +62,8 @@ end NodeEventMetadata object NodeEventMetadata: final val empty = NodeEventMetadata() def apply(anchor: Anchor): NodeEventMetadata = NodeEventMetadata(anchor = Some(anchor)) - @scala.annotation.targetName("applyForTag") + def apply(anchor: Anchor, tag: Tag): NodeEventMetadata = + NodeEventMetadata(anchor = Some(anchor), tag = Some(tag)) def apply(tag: Tag): NodeEventMetadata = NodeEventMetadata(tag = Some(tag)) opaque type Anchor = String diff --git a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala index 74f0b07c0..dfb835d60 100644 --- a/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala +++ b/yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala @@ -6,6 +6,8 @@ import scala.util.Failure import scala.util.Success import scala.util.Try +import org.virtuslab.yaml.CoreSchemaTag +import org.virtuslab.yaml.CustomTag import org.virtuslab.yaml.ParseError import org.virtuslab.yaml.Range import org.virtuslab.yaml.Tag @@ -388,7 +390,7 @@ final class ParserImpl private (in: Tokenizer) extends Parser: case _ => Right( Event( - EventKind.Scalar("", ScalarStyle.Plain, metadata), + EventKind.Scalar("", ScalarStyle.Plain, metadata.withTag(Tag.nullTag)), nextToken.range ) ) @@ -407,14 +409,18 @@ final class ParserImpl private (in: Tokenizer) extends Parser: in.popToken() value match case TagValue.NonSpecific => - parseNodeAttributes(in.peekToken(), metadata.withTag(Tag("!"))) + parseNodeAttributes(in.peekToken(), metadata) case TagValue.Verbatim(value) => - parseNodeAttributes(in.peekToken(), metadata.withTag(Tag(value))) + parseNodeAttributes(in.peekToken(), metadata.withTag(CustomTag(value))) case TagValue.Shorthand(handle, suffix) => val handleKey = handle.value directives.get(handleKey) match case Some(prefix) => - parseNodeAttributes(in.peekToken(), metadata.withTag(Tag(s"$prefix$suffix"))) + val tagValue = s"$prefix$suffix" + val tag = + if Tag.coreSchemaValues.contains(tagValue) then CoreSchemaTag(tagValue) + else CustomTag(tagValue) + parseNodeAttributes(in.peekToken(), metadata.withTag(tag)) case None => Left(ParseError(s"There is no registered tag directive for handle $handleKey")) case _ => Right(metadata, token) diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/ComposerSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/ComposerSuite.scala index 36f7a706a..658828b3e 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/ComposerSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/ComposerSuite.scala @@ -34,7 +34,7 @@ class ComposerSuite extends munit.FunSuite: assertEquals(ComposerImpl.fromEvents(events), expected) } - test("mapping of scalars".only) { + test("mapping of scalars") { val events = List( StreamStart, DocumentStart(), diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala index 504a2f235..bcf6ce71b 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/ConstructSuite.scala @@ -46,7 +46,7 @@ class ConstructSuite extends munit.FunSuite: assertEquals(node.as[DummyClass], expectedConstructError) } - test("decode as Any".only) { + test("decode as Any") { val node = MappingNode( SequenceNode(1, 2) -> ScalarNode("seq") @@ -56,8 +56,5 @@ class ConstructSuite extends munit.FunSuite: Seq(1, 2) -> "seq" ) - println(SequenceNode(1, 2)) - println(node.as[Any]) - assertEquals(node.as[Any], Right(expected)) } diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderSuite.scala index d7e1cf28e..8a0139290 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/decoder/DecoderSuite.scala @@ -205,7 +205,7 @@ class DecoderSuite extends munit.FunSuite: assertEquals(yaml.as[Map[Any, Any]], Right(expected)) } - test("decode using custom tag".only) { + test("decode using custom tag") { case class Custom(x: Int, doubledX: Int) val yaml = @@ -214,19 +214,13 @@ class DecoderSuite extends munit.FunSuite: val expected = Custom(5, 10) - val decoder = new YamlDecoder[Custom] { - override def construct(node: Node)(using - settings: LoadSettings = LoadSettings.empty - ): Either[ConstructError, Custom] = node match { - case ScalarNode(value, _) => - val int = value.toInt - Right(Custom(int, int * 2)) - case _ => ??? - } - }.asInstanceOf[YamlDecoder[Any]] + val decoder = YamlDecoder[Custom] { case ScalarNode(value, _) => + val int = value.toInt + Right(Custom(int, int * 2)) + } given settings: LoadSettings = LoadSettings( - Map(Tag("!Custom") -> decoder) + Map(CustomTag("!Custom") -> decoder) ) assertEquals(yaml.as[Any], Right(expected)) diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/AnchorSpec.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/AnchorSpec.scala index 01c390342..51f596727 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/AnchorSpec.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/AnchorSpec.scala @@ -89,7 +89,7 @@ class AnchorSpec extends BaseYamlSuite: DocumentStart(explicit = true), MappingStart(), Scalar("a"), - Scalar("", metadata = NodeEventMetadata(Anchor("anchor"))), + Scalar("", metadata = NodeEventMetadata(Anchor("anchor"), Tag.nullTag)), Scalar("b"), Alias(Anchor("anchor")), MappingEnd, diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/MappingSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/MappingSuite.scala index 1b1a13a7d..6854223ce 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/MappingSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/MappingSuite.scala @@ -2,6 +2,7 @@ package org.virtuslab.yaml package parser import org.virtuslab.yaml.internal.load.parse.EventKind.* +import org.virtuslab.yaml.internal.load.parse.NodeEventMetadata import org.virtuslab.yaml.internal.load.reader.token.ScalarStyle class MappingSuite extends BaseYamlSuite: @@ -151,7 +152,7 @@ class MappingSuite extends BaseYamlSuite: DocumentStart(), MappingStart(), Scalar("key"), - Scalar(""), + Scalar("", metadata = NodeEventMetadata(Tag.nullTag)), Scalar("key2"), Scalar("value"), MappingEnd, @@ -172,7 +173,7 @@ class MappingSuite extends BaseYamlSuite: DocumentStart(), MappingStart(), Scalar("key"), - Scalar(""), + Scalar("", metadata = NodeEventMetadata(Tag.nullTag)), Scalar("period"), Scalar("10"), MappingEnd, @@ -437,13 +438,13 @@ class MappingSuite extends BaseYamlSuite: SequenceStart(), FlowMappingStart(), Scalar("single line", ScalarStyle.DoubleQuoted), - Scalar(""), + Scalar("", metadata = NodeEventMetadata(Tag.nullTag)), Scalar("a"), Scalar("b"), FlowMappingEnd, FlowMappingStart(), Scalar("multi line", ScalarStyle.DoubleQuoted), - Scalar(""), + Scalar("", metadata = NodeEventMetadata(Tag.nullTag)), Scalar("a"), Scalar("b"), FlowMappingEnd, diff --git a/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/TagSuite.scala b/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/TagSuite.scala index 46e3a704a..4d01d7457 100644 --- a/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/TagSuite.scala +++ b/yaml/shared/src/test/scala/org/virtuslab/yaml/parser/TagSuite.scala @@ -1,7 +1,6 @@ package org.virtuslab.yaml package parser -import org.virtuslab.yaml.Tag import org.virtuslab.yaml.internal.load.parse.Anchor import org.virtuslab.yaml.internal.load.parse.EventKind import org.virtuslab.yaml.internal.load.parse.EventKind.* @@ -22,13 +21,13 @@ class TagSuite extends BaseYamlSuite: val expectedEvents = List( StreamStart, DocumentStart(), - Scalar("bar", ScalarStyle.DoubleQuoted, NodeEventMetadata(Tag("!foo"))), + Scalar("bar", ScalarStyle.DoubleQuoted, NodeEventMetadata(CustomTag("!foo"))), DocumentEnd(explicit = true), DocumentStart(explicit = true), Scalar( "bar", ScalarStyle.DoubleQuoted, - NodeEventMetadata(Tag("tag:example.com,2000:app/foo")) + NodeEventMetadata(CustomTag("tag:example.com,2000:app/foo")) ), DocumentEnd(), StreamEnd @@ -46,13 +45,13 @@ class TagSuite extends BaseYamlSuite: val expectedEvents = List( StreamStart, DocumentStart(), - Scalar("bar", ScalarStyle.DoubleQuoted, NodeEventMetadata(Tag("!foo"))), + Scalar("bar", ScalarStyle.DoubleQuoted, NodeEventMetadata(CustomTag("!foo"))), DocumentEnd(), DocumentStart(explicit = true), Scalar( "bar", ScalarStyle.DoubleQuoted, - NodeEventMetadata(Tag("tag:example.com,2000:app/foo")) + NodeEventMetadata(CustomTag("tag:example.com,2000:app/foo")) ), DocumentEnd(), StreamEnd @@ -72,9 +71,9 @@ class TagSuite extends BaseYamlSuite: StreamStart, DocumentStart(explicit = true), SequenceStart(), - Scalar("foo", metadata = NodeEventMetadata(Tag("!local"))), - Scalar("bar", metadata = NodeEventMetadata(Tag("tag:yaml.org,2002:str"))), - Scalar("baz", metadata = NodeEventMetadata(Tag("tag:example.com,2000:app/tag!"))), + Scalar("foo", metadata = NodeEventMetadata(CustomTag("!local"))), + Scalar("bar", metadata = NodeEventMetadata(CoreSchemaTag("tag:yaml.org,2002:str"))), + Scalar("baz", metadata = NodeEventMetadata(CustomTag("tag:example.com,2000:app/tag!"))), SequenceEnd, DocumentEnd(), StreamEnd @@ -89,7 +88,7 @@ class TagSuite extends BaseYamlSuite: val expectedEvents = List( StreamStart, DocumentStart(), - Scalar("a", metadata = NodeEventMetadata(Tag("!"))), + Scalar("a"), DocumentEnd(), StreamEnd ) @@ -106,7 +105,7 @@ class TagSuite extends BaseYamlSuite: val expectedEvents = List( StreamStart, DocumentStart(explicit = true), - SequenceStart(NodeEventMetadata(Tag("tag:yaml.org,2002:omap"))), + SequenceStart(NodeEventMetadata(CustomTag("tag:yaml.org,2002:omap"))), MappingStart(), Scalar("Mark McGwire"), Scalar("65"), @@ -139,7 +138,7 @@ class TagSuite extends BaseYamlSuite: Scalar( "bar", ScalarStyle.DoubleQuoted, - NodeEventMetadata(Tag("tag:example.com,2000:app/foo")) + NodeEventMetadata(CustomTag("tag:example.com,2000:app/foo")) ), SequenceEnd, DocumentEnd(),