Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix parsing anchor in mapping block #91

Merged
merged 3 commits into from
Nov 8, 2021
Merged

Fix parsing anchor in mapping block #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
12490e4
Fix parsing anchor in mapping block
lwronski Nov 4, 2021
4e332f6
Fix parsing anchor in flow collection
lwronski Nov 7, 2021
b414eb6
Minor changes
lwronski Nov 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/parse/ParserImpl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ final class ParserImpl private (in: Tokenizer) extends Parser:
case TokenKind.FlowSequenceStart =>
productions.prependAll(ParseFlowNode :: Nil)
getNextEventImpl()
case TokenKind.Scalar(_, _) =>
case TokenKind.Scalar(_, _) | _: TokenKind.Anchor =>
productions.prependAll(
ParseFlowNode :: ParseFlowMappingComma :: ParseFlowMappingEntry :: Nil
)
Expand Down Expand Up @@ -292,7 +292,7 @@ final class ParserImpl private (in: Tokenizer) extends Parser:

def parseFlowSeqEntryOpt() = token.kind match
case TokenKind.FlowMappingStart | TokenKind.FlowSequenceStart | _: TokenKind.Scalar |
TokenKind.MappingKey =>
_: TokenKind.Alias | TokenKind.MappingKey =>
productions.prependAll(ParseFlowSeqEntry :: Nil)
getNextEventImpl()
case _ =>
Expand Down Expand Up @@ -332,6 +332,11 @@ final class ParserImpl private (in: Tokenizer) extends Parser:
case TokenKind.Scalar(value, style) =>
in.popToken()
Right(Event(EventKind.Scalar(value, style, NodeEventMetadata(anchor)), pos))
case TokenKind.Alias(alias) =>
if anchor.isDefined then Left(ParseError.from("Alias cannot have an anchor", nextToken))
else
in.popToken()
Right(Event(EventKind.Alias(Anchor(alias)), nextToken.pos))
case _ =>
Left(ParseError.from(TokenKind.Scalar.toString, token))

Expand Down
17 changes: 8 additions & 9 deletions yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/reader/ReaderCtx.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@ case class ReaderCtx(reader: Reader) {

def indent: Int = indentations.lastOption.getOrElse(-1)
def addIndent(newIndent: Int): Unit = indentations.append(newIndent)
def removeLastIndent(): Unit = if (indentations.nonEmpty) indentations.removeLast()

def checkIndents(current: Int): Unit =
def checkIndents(current: Int): List[Token] =
if current < indent then
indentations.removeLast()
tokens.append(Token(BlockEnd, reader.pos))
checkIndents(current)
Token(BlockEnd, reader.pos) +: checkIndents(current)
else Nil

def enterFlowSequence: Unit = flowSequenceLevel += 1
def leaveFlowSequence: Unit = flowSequenceLevel -= 1
Expand All @@ -40,13 +41,11 @@ case class ReaderCtx(reader: Reader) {
def isInFlowSequence: Boolean = flowSequenceLevel > 0
def isInFlowCollection: Boolean = isInFlowMapping || isInFlowSequence

def parseDocumentStart(indent: Int): Token =
checkIndents(-1)
Token(DocumentStart, reader.pos)
def parseDocumentStart(indent: Int): List[Token] =
checkIndents(-1) ++ List(Token(DocumentStart, reader.pos))

def parseDocumentEnd(): Token =
checkIndents(-1)
Token(DocumentEnd, reader.pos)
def parseDocumentEnd(): List[Token] =
checkIndents(-1) ++ List(Token(DocumentEnd, reader.pos))
}

object ReaderCtx:
Expand Down
75 changes: 46 additions & 29 deletions yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/reader/Tokenizer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
override def popToken(): Token = ctx.tokens.removeHead()

private def getToken(): Token =
ctx.tokens.append(getNextTokens())
ctx.tokens.appendAll(getNextTokens())
ctx.tokens.head

private def getNextTokens(): Token =
private def getNextTokens(): List[Token] =
skipUntilNextToken()
ctx.checkIndents(in.column)
val peeked = in.peek()
peeked match
val closedTokens = ctx.checkIndents(in.column)
val peeked = in.peek()
val tokens = peeked match
case Some('-') if isDocumentStart => parseDocumentStart()
case Some('-') if in.isNextWhitespace => parseBlockSequence()
case Some('.') if isDocumentEnd => parseDocumentEnd()
Expand All @@ -41,14 +41,14 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
case Some('{') => parseFlowMappingStart()
case Some('}') => parseFlowMappingEnd()
case Some('&') => parseAnchor()
case Some('*') => parseAlias()
case Some(',') =>
in.skipCharacter()
Token(Comma, in.pos)
List(Token(Comma, in.pos))
case Some(_) => fetchValue()
case None =>
ctx.checkIndents(-1)
Token(StreamEnd, in.pos)
ctx.checkIndents(-1) ++ List(Token(StreamEnd, in.pos))

closedTokens ++ tokens

private def isDocumentStart =
in.peekN(3) == "---" && in.peek(3).exists(_.isWhitespace)
Expand All @@ -67,30 +67,30 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
private def parseFlowSequenceStart() =
in.skipCharacter()
ctx.enterFlowSequence
Token(FlowSequenceStart, in.pos)
List(Token(FlowSequenceStart, in.pos))

private def parseFlowSequenceEnd() =
in.skipCharacter()
ctx.leaveFlowSequence
Token(FlowSequenceEnd, in.pos)
List(Token(FlowSequenceEnd, in.pos))

private def parseFlowMappingStart() =
in.skipCharacter()
ctx.enterFlowMapping
Token(FlowMappingStart, in.pos)
List(Token(FlowMappingStart, in.pos))

private def parseFlowMappingEnd() =
in.skipCharacter()
ctx.leaveFlowMapping
Token(FlowMappingEnd, in.pos)
List(Token(FlowMappingEnd, in.pos))

private def parseBlockSequence() =
if (!ctx.isInFlowCollection && ctx.indent < in.column) then
ctx.addIndent(in.column)
Token(SequenceStart, in.pos)
List(Token(SequenceStart, in.pos))
else
in.skipCharacter()
Token(SequenceValue, in.pos)
List(Token(SequenceValue, in.pos))

private def parseAnchorName(): (String, Position) =
val invalidChars = Set('[', ']', '{', '}', ',')
Expand All @@ -109,9 +109,22 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
val name = readAnchorName()
(name, pos)

private def parseAnchor() =
val (name, pos) = parseAnchorName()
Token(Anchor(name), pos)
private def parseAnchor(): List[Token] =
val (name, anchorPos) = parseAnchorName()
val nexTokens = getNextTokens()

val anchorToken = Token(Anchor(name), anchorPos)
nexTokens match {
case (Token(_: MappingStart.type, _) |
Token(_: FlowMappingStart.type, _)) :: Token(_: MappingKey.type, _) :: rest =>
ctx.removeLastIndent()
ctx.addIndent(anchorPos.column)
lwronski marked this conversation as resolved.
Show resolved Hide resolved
nexTokens.take(2) ::: anchorToken +: rest
case Token(_: MappingKey.type, _) :: rest
if (ctx.indent == anchorPos.column || ctx.isInFlowCollection) =>
nexTokens.take(1) ::: anchorToken +: rest
lwronski marked this conversation as resolved.
Show resolved Hide resolved
case _ => List(anchorToken) ::: nexTokens
}

private def parseAlias() =
val (name, pos) = parseAnchorName()
Expand Down Expand Up @@ -305,33 +318,37 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
Token(Scalar(scalar.trim, ScalarStyle.Plain), pos)
}

private def fetchValue(): Token =
private def fetchValue(): List[Token] =
skipUntilNextToken()
val peeked = in.peek()
val scalar: Token = peeked match
case Some('"') => parseDoubleQuoteValue()
case Some('\'') => parseSingleQuoteValue()
case Some('>') => parseFoldedValue()
case Some('|') => parseLiteral()
case Some('*') => parseAlias()
case _ => parseScalarValue()

skipUntilNextToken()
val peeked2 = in.peek()
peeked2 match
case Some(':') =>
in.skipCharacter()
if (ctx.indent < scalar.pos.column && !ctx.isInFlowCollection) then
ctx.addIndent(scalar.pos.column)
ctx.tokens.appendAll(List(Token(MappingStart, scalar.pos)))

ctx.tokens.appendAll(
List(
Token(MappingKey, scalar.pos),
scalar
val maybeMappingStart =
if (ctx.indent < scalar.pos.column && !ctx.isInFlowCollection) then
ctx.addIndent(scalar.pos.column)
List(Token(MappingStart, scalar.pos))
else Nil

maybeMappingStart ++ List(
Token(MappingKey, scalar.pos),
scalar,
Token(
MappingValue,
scalar.pos
)
)
Token(MappingValue, scalar.pos)
case _ => scalar
case _ => List(scalar)

def skipUntilNextToken(): Unit =
while (in.isWhitespace) do in.skipCharacter()
Expand Down
49 changes: 22 additions & 27 deletions yaml/shared/src/test/scala/org/virtuslab/yaml/parser/AnchorSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,7 @@ class AnchorSpec extends BaseYamlSuite:
assertEquals(yaml.events, Right(expectedEvents))
}

// need improvement in tokenizer
test("in mapping but with keys aliased".ignore) {
test("in mapping but with keys aliased") {
val yaml =
s"""|&a a: &b b
|*b : *a
Expand All @@ -48,8 +47,8 @@ class AnchorSpec extends BaseYamlSuite:
MappingStart(),
Scalar("a", metadata = NodeEventMetadata(Anchor("a"))),
Scalar("b", metadata = NodeEventMetadata(Anchor("b"))),
Alias(Anchor("a")),
Alias(Anchor("b")),
Alias(Anchor("a")),
MappingEnd,
DocumentEnd(),
StreamEnd
Expand Down Expand Up @@ -102,11 +101,11 @@ class AnchorSpec extends BaseYamlSuite:
assertEquals(yaml.events, Right(expectedEvents))
}

test("anchor in flow collections".ignore) {
test("anchor in flow collections") {
val yaml =
s"""|{
| a : &b b,
| seq: [a, *b]
| &a a : &b b,
| seq: [*a, *b]
|}""".stripMargin

val expectedEvents = List(
Expand All @@ -128,37 +127,33 @@ class AnchorSpec extends BaseYamlSuite:
assertEquals(yaml.events, Right(expectedEvents))
}

test("anchor & alias".ignore) {
val yaml =
s"""|---
|a: &anchor
|b: *anchor
|""".stripMargin

val expectedEvents = List(
StreamStart,
DocumentStart(),
DocumentEnd(),
StreamEnd
)
assertEquals(yaml.events, Right(expectedEvents))
}

test("anchor & alias".ignore) {
test("anchor & alias") {
val yaml =
s"""|---
|hr:
| - Mark McGwire
| # Following node labeled SS
| - &SS Sammy Sosa
| # Following node labeled anchor
| - &anchor Sammy Sosa
|rbi:
| - *SS # Subsequent occurrence
| - *anchor # Subsequent occurrence
| - Ken Griffey
|""".stripMargin

val expectedEvents = List(
StreamStart,
DocumentStart(),
DocumentStart(explicit = true),
MappingStart(),
Scalar("hr"),
SequenceStart(),
Scalar("Mark McGwire"),
Scalar("Sammy Sosa", metadata = NodeEventMetadata(Anchor("anchor"))),
SequenceEnd,
Scalar("rbi"),
SequenceStart(),
Alias(Anchor("anchor")),
Scalar("Ken Griffey"),
SequenceEnd,
MappingEnd,
DocumentEnd(),
StreamEnd
)
Expand Down