Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix parsing anchor in mapping block #91

Merged
merged 3 commits into from
Nov 8, 2021
Merged

Fix parsing anchor in mapping block #91

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
12490e4
Fix parsing anchor in mapping block
lwronski Nov 4, 2021
4e332f6
Fix parsing anchor in flow collection
lwronski Nov 7, 2021
b414eb6
Minor changes
lwronski Nov 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 7 additions & 2 deletions yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ final class ParserImpl private (in: Tokenizer) extends Parser:
case TokenKind.FlowSequenceStart =>
productions.prependAll(ParseFlowNode :: Nil)
getNextEventImpl()
case TokenKind.Scalar(_, _) =>
case TokenKind.Scalar(_, _) | _: TokenKind.Anchor =>
productions.prependAll(
ParseFlowNode :: ParseFlowMappingComma :: ParseFlowMappingEntry :: Nil
)
Expand Down Expand Up @@ -292,7 +292,7 @@ final class ParserImpl private (in: Tokenizer) extends Parser:

def parseFlowSeqEntryOpt() = token.kind match
case TokenKind.FlowMappingStart | TokenKind.FlowSequenceStart | _: TokenKind.Scalar |
TokenKind.MappingKey =>
_: TokenKind.Alias | TokenKind.MappingKey =>
productions.prependAll(ParseFlowSeqEntry :: Nil)
getNextEventImpl()
case _ =>
Expand Down Expand Up @@ -332,6 +332,11 @@ final class ParserImpl private (in: Tokenizer) extends Parser:
case TokenKind.Scalar(value, style) =>
in.popToken()
Right(Event(EventKind.Scalar(value, style, NodeEventMetadata(anchor)), pos))
case TokenKind.Alias(alias) =>
if anchor.isDefined then Left(ParseError.from("Alias cannot have an anchor", nextToken))
else
in.popToken()
Right(Event(EventKind.Alias(Anchor(alias)), nextToken.pos))
case _ =>
Left(ParseError.from(TokenKind.Scalar.toString, token))

Expand Down
9 changes: 5 additions & 4 deletions yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/reader/ReaderCtx.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ case class ReaderCtx(reader: Reader) {

def indent: Int = indentations.lastOption.getOrElse(-1)
def addIndent(newIndent: Int): Unit = indentations.append(newIndent)
def removeLastIndent(): Unit = indentations.removeLast()
lwronski marked this conversation as resolved.
Show resolved Hide resolved

def checkIndents(current: Int): Unit =
lwronski marked this conversation as resolved.
Show resolved Hide resolved
if current < indent then
Expand All @@ -40,13 +41,13 @@ case class ReaderCtx(reader: Reader) {
def isInFlowSequence: Boolean = flowSequenceLevel > 0
def isInFlowCollection: Boolean = isInFlowMapping || isInFlowSequence

def parseDocumentStart(indent: Int): Token =
def parseDocumentStart(indent: Int): List[Token] =
checkIndents(-1)
Token(DocumentStart, reader.pos)
List(Token(DocumentStart, reader.pos))

def parseDocumentEnd(): Token =
def parseDocumentEnd(): List[Token] =
checkIndents(-1)
Token(DocumentEnd, reader.pos)
List(Token(DocumentEnd, reader.pos))
}

object ReaderCtx:
Expand Down
64 changes: 38 additions & 26 deletions yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/reader/Tokenizer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
override def popToken(): Token = ctx.tokens.removeHead()

private def getToken(): Token =
ctx.tokens.append(getNextTokens())
ctx.tokens.appendAll(getNextTokens())
ctx.tokens.head

private def getNextTokens(): Token =
private def getNextTokens(): List[Token] =
skipUntilNextToken()
ctx.checkIndents(in.column)
val peeked = in.peek()
Expand All @@ -41,14 +41,13 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
case Some('{') => parseFlowMappingStart()
case Some('}') => parseFlowMappingEnd()
case Some('&') => parseAnchor()
case Some('*') => parseAlias()
case Some(',') =>
in.skipCharacter()
Token(Comma, in.pos)
List(Token(Comma, in.pos))
case Some(_) => fetchValue()
case None =>
ctx.checkIndents(-1)
Token(StreamEnd, in.pos)
List(Token(StreamEnd, in.pos))

private def isDocumentStart =
in.peekN(3) == "---" && in.peek(3).exists(_.isWhitespace)
Expand All @@ -67,30 +66,30 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
private def parseFlowSequenceStart() =
in.skipCharacter()
ctx.enterFlowSequence
Token(FlowSequenceStart, in.pos)
List(Token(FlowSequenceStart, in.pos))

private def parseFlowSequenceEnd() =
in.skipCharacter()
ctx.leaveFlowSequence
Token(FlowSequenceEnd, in.pos)
List(Token(FlowSequenceEnd, in.pos))

private def parseFlowMappingStart() =
in.skipCharacter()
ctx.enterFlowMapping
Token(FlowMappingStart, in.pos)
List(Token(FlowMappingStart, in.pos))

private def parseFlowMappingEnd() =
in.skipCharacter()
ctx.leaveFlowMapping
Token(FlowMappingEnd, in.pos)
List(Token(FlowMappingEnd, in.pos))

private def parseBlockSequence() =
if (!ctx.isInFlowCollection && ctx.indent < in.column) then
ctx.addIndent(in.column)
Token(SequenceStart, in.pos)
List(Token(SequenceStart, in.pos))
else
in.skipCharacter()
Token(SequenceValue, in.pos)
List(Token(SequenceValue, in.pos))

private def parseAnchorName(): (String, Position) =
val invalidChars = Set('[', ']', '{', '}', ',')
Expand All @@ -109,9 +108,22 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
val name = readAnchorName()
(name, pos)

private def parseAnchor() =
val (name, pos) = parseAnchorName()
Token(Anchor(name), pos)
private def parseAnchor(): List[Token] =
val (name, anchorPos) = parseAnchorName()
val nexTokens = getNextTokens()

val anchorToken = Token(Anchor(name), anchorPos)
nexTokens match {
case (Token(_: MappingStart.type, _) |
Token(_: FlowMappingStart.type, _)) :: Token(_: MappingKey.type, _) :: rest =>
ctx.removeLastIndent()
ctx.addIndent(anchorPos.column)
lwronski marked this conversation as resolved.
Show resolved Hide resolved
nexTokens.take(2) ::: anchorToken +: rest
case Token(_: MappingKey.type, _) :: rest
if (ctx.indent == anchorPos.column || ctx.isInFlowCollection) =>
nexTokens.take(1) ::: anchorToken +: rest
lwronski marked this conversation as resolved.
Show resolved Hide resolved
case _ => List(anchorToken) ::: nexTokens
}

private def parseAlias() =
val (name, pos) = parseAnchorName()
Expand Down Expand Up @@ -305,33 +317,33 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
Token(Scalar(scalar.trim, ScalarStyle.Plain), pos)
}

private def fetchValue(): Token =
private def fetchValue(): List[Token] =
skipUntilNextToken()
val peeked = in.peek()
val scalar: Token = peeked match
case Some('"') => parseDoubleQuoteValue()
case Some('\'') => parseSingleQuoteValue()
case Some('>') => parseFoldedValue()
case Some('|') => parseLiteral()
case Some('*') => parseAlias()
case _ => parseScalarValue()

skipUntilNextToken()
val peeked2 = in.peek()
peeked2 match
case Some(':') =>
in.skipCharacter()
if (ctx.indent < scalar.pos.column && !ctx.isInFlowCollection) then
ctx.addIndent(scalar.pos.column)
ctx.tokens.appendAll(List(Token(MappingStart, scalar.pos)))

ctx.tokens.appendAll(
List(
Token(MappingKey, scalar.pos),
scalar
)
val maybeMappingStart =
if (ctx.indent < scalar.pos.column && !ctx.isInFlowCollection) then
ctx.addIndent(scalar.pos.column)
List(Token(MappingStart, scalar.pos))
else Nil

maybeMappingStart :+ Token(MappingKey, scalar.pos) :+ scalar :+ Token(
MappingValue,
scalar.pos
lwronski marked this conversation as resolved.
Show resolved Hide resolved
)
Token(MappingValue, scalar.pos)
case _ => scalar
case _ => List(scalar)

def skipUntilNextToken(): Unit =
while (in.isWhitespace) do in.skipCharacter()
Expand Down
43 changes: 19 additions & 24 deletions yaml/shared/src/test/scala/org/virtuslab/yaml/parser/AnchorSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,7 @@ class AnchorSpec extends BaseYamlSuite:
assertEquals(yaml.events, Right(expectedEvents))
}

// need improvement in tokenizer
test("in mapping but with keys aliased".ignore) {
test("in mapping but with keys aliased") {
val yaml =
s"""|&a a: &b b
|*b : *a
Expand All @@ -48,8 +47,8 @@ class AnchorSpec extends BaseYamlSuite:
MappingStart(),
Scalar("a", metadata = NodeEventMetadata(Anchor("a"))),
Scalar("b", metadata = NodeEventMetadata(Anchor("b"))),
Alias(Anchor("a")),
Alias(Anchor("b")),
Alias(Anchor("a")),
MappingEnd,
DocumentEnd(),
StreamEnd
Expand Down Expand Up @@ -102,11 +101,11 @@ class AnchorSpec extends BaseYamlSuite:
assertEquals(yaml.events, Right(expectedEvents))
}

test("anchor in flow collections".ignore) {
test("anchor in flow collections") {
val yaml =
s"""|{
| a : &b b,
| seq: [a, *b]
| &a a : &b b,
| seq: [*a, *b]
|}""".stripMargin

val expectedEvents = List(
Expand All @@ -128,23 +127,7 @@ class AnchorSpec extends BaseYamlSuite:
assertEquals(yaml.events, Right(expectedEvents))
}

test("anchor & alias".ignore) {
val yaml =
s"""|---
|a: &anchor
|b: *anchor
|""".stripMargin

val expectedEvents = List(
StreamStart,
DocumentStart(),
DocumentEnd(),
StreamEnd
)
assertEquals(yaml.events, Right(expectedEvents))
}

test("anchor & alias".ignore) {
test("anchor & alias") {
val yaml =
s"""|---
|hr:
Expand All @@ -158,7 +141,19 @@ class AnchorSpec extends BaseYamlSuite:

val expectedEvents = List(
StreamStart,
DocumentStart(),
DocumentStart(explicit = true),
MappingStart(),
Scalar("hr"),
SequenceStart(),
Scalar("Mark McGwire"),
Scalar("Sammy Sosa", metadata = NodeEventMetadata(Anchor("SS"))),
SequenceEnd,
Scalar("rbi"),
SequenceStart(),
Alias(Anchor("SS")),
lwronski marked this conversation as resolved.
Show resolved Hide resolved
Scalar("Ken Griffey"),
SequenceEnd,
MappingEnd,
DocumentEnd(),
StreamEnd
)
Expand Down