Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Parse new lines for plain scalar style #66

Merged
merged 2 commits into from
Sep 7, 2021
Merged

Parse new lines for plain scalar style #66

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
73f93aa
Parse new lines for plain scalar style
lwronski Sep 2, 2021
db4d383
Fixes after review
lwronski Sep 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 16 additions & 9 deletions yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/reader/ReaderCtx.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ package org.virtuslab.yaml.internal.load.reader

import org.virtuslab.yaml.Position
import org.virtuslab.yaml.internal.load.reader.Reader
import org.virtuslab.yaml.internal.load.reader.ReaderState.ReaderStateWithIndent
import org.virtuslab.yaml.internal.load.reader.StringReader

import scala.annotation.tailrec
import scala.collection.mutable

import token.Token
case class ReaderCtx(
stateStack: mutable.Stack[ReaderState],
Expand Down Expand Up @@ -38,10 +38,16 @@ case class ReaderCtx(
closeOpenedCollectionMapping(indent)
case _ => ()

def getIndentOfLatestCollection(): Option[Int] =
stateStack.headOption match {
case Some(state: ReaderStateWithIndent) => Some(state.indent)
case _ => None
}

def appendState(state: ReaderState): Unit = stateStack.push(state)

def closeOpenedFlowMapping(): List[Token] = stateStack.headOption match
case Some(ReaderState.FlowMapping) =>
case Some(ReaderState.FlowMapping(_)) =>
stateStack.pop()
List(Token.FlowMappingEnd(reader.pos()))
case _ =>
Expand All @@ -52,7 +58,7 @@ case class ReaderCtx(
case Some(ReaderState.Sequence(_)) =>
stateStack.pop()
List(Token.SequenceEnd(reader.pos()))
case Some(ReaderState.FlowSequence) =>
case Some(ReaderState.FlowSequence(_)) =>
stateStack.pop()
List(Token.FlowSequenceEnd(reader.pos()))
case _ =>
Expand All @@ -70,15 +76,16 @@ case class ReaderCtx(

def isAllowedSpecialCharacter(char: Char): Boolean =
stateStack.headOption match
case Some(ReaderState.FlowMapping) if char == '}' => false
case Some(ReaderState.FlowMapping) | Some(ReaderState.FlowSequence) if char == ',' => false
case Some(ReaderState.FlowSequence) if char == ']' => false
case _ => true
case Some(ReaderState.FlowMapping(_)) if char == '}' => false
case Some(ReaderState.FlowMapping(_)) | Some(ReaderState.FlowSequence(_)) if char == ',' =>
false
case Some(ReaderState.FlowSequence(_)) if char == ']' => false
case _ => true

def isFlowMapping(): Boolean =
stateStack.headOption match
case Some(ReaderState.FlowMapping) => true
case _ => false
case Some(ReaderState.FlowMapping(_)) => true
case _ => false

def closeOpenedScopes(): List[Token] =
@tailrec
Expand Down
14 changes: 8 additions & 6 deletions yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/reader/ReaderState.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ package org.virtuslab.yaml.internal.load.reader

sealed trait ReaderState
case object ReaderState:
case object Stream extends ReaderState
case object Document extends ReaderState
final case class Mapping(indent: Int) extends ReaderState
final case class Sequence(indent: Int) extends ReaderState
case object FlowMapping extends ReaderState
case object FlowSequence extends ReaderState
case object Stream extends ReaderState
case object Document extends ReaderState
lwronski marked this conversation as resolved.
Show resolved Hide resolved
sealed trait ReaderStateWithIndent extends ReaderState:
def indent: Int
final case class Mapping(indent: Int) extends ReaderStateWithIndent
final case class Sequence(indent: Int) extends ReaderStateWithIndent
final case class FlowMapping(indent: Int) extends ReaderStateWithIndent
final case class FlowSequence(indent: Int) extends ReaderStateWithIndent
63 changes: 29 additions & 34 deletions yaml/shared/src/main/scala/org/virtuslab/yaml/internal/load/reader/Tokenizer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ trait Tokenizer:

private[yaml] class Scanner(str: String) extends Tokenizer {

private val ctx = ReaderCtx.init(str)
private val in = ctx.reader
private var indent = 0
private val ctx = ReaderCtx.init(str)
private val in = ctx.reader

override def peekToken(): Token = ctx.tokens.headOption match
case Some(token) => token
Expand Down Expand Up @@ -58,7 +57,7 @@ private[yaml] class Scanner(str: String) extends Tokenizer {

private def parseFlowSequenceStart() =
in.skipCharacter()
ctx.appendState(ReaderState.FlowSequence)
ctx.appendState(ReaderState.FlowSequence(in.column))
List(FlowSequenceStart(in.pos()))

private def parseFlowSequenceEnd() =
Expand All @@ -67,21 +66,20 @@ private[yaml] class Scanner(str: String) extends Tokenizer {

private def parseFlowMappingStart() =
in.skipCharacter()
ctx.appendState(ReaderState.FlowMapping)
ctx.appendState(ReaderState.FlowMapping(in.column))
List(FlowMappingStart(in.pos()))

private def parseFlowMappingEnd() =
in.skipCharacter()
ctx.closeOpenedFlowMapping()

private def parseBlockSequence() =
ctx.closeOpenedCollectionSequences(indent)
if (ctx.shouldParseSequenceEntry(indent)) then
ctx.closeOpenedCollectionSequences(in.column)
if (ctx.shouldParseSequenceEntry(in.column)) then
in.skipCharacter()
indent += 1
getNextTokens()
else
ctx.appendState(ReaderState.Sequence(indent))
ctx.appendState(ReaderState.Sequence(in.column))
List(SequenceStart(in.pos()))

private def parseDoubleQuoteValue(): Token =
Expand Down Expand Up @@ -111,13 +109,11 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
*/
private def parseBlockHeader(): Unit =
while (in.peek() == Some(' ')) {
indent += 1
in.skipCharacter()
}

if in.isNewline then
in.skipCharacter()
indent = 0
parseBlockHeader()

/**
Expand All @@ -142,7 +138,7 @@ private[yaml] class Scanner(str: String) extends Tokenizer {

parseBlockHeader()

val foldedIndent = indent
val foldedIndent = in.column
skipUntilNextIndent(foldedIndent)

@tailrec
Expand All @@ -151,7 +147,7 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
case Some('\n') =>
sb.append(in.read())
skipUntilNextIndent(foldedIndent)
if (!in.isWhitespace && indent != foldedIndent) then sb.result()
if (!in.isWhitespace && in.column != foldedIndent) then sb.result()
else readLiteral()
case Some(char) =>
sb.append(in.read())
Expand All @@ -170,7 +166,7 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
val chompingIndicator = parseChompingIndicator()

parseBlockHeader()
val foldedIndent = indent
val foldedIndent = in.column
skipUntilNextIndent(foldedIndent)

def chompedEmptyLines() =
Expand All @@ -192,7 +188,7 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
} else {
in.skipCharacter()
skipUntilNextIndent(foldedIndent)
if (!in.isWhitespace && indent != foldedIndent) then sb.result()
if (!in.isWhitespace && in.column != foldedIndent) then sb.result()
else
sb.append(" ")
readFolded()
Expand Down Expand Up @@ -235,22 +231,27 @@ private[yaml] class Scanner(str: String) extends Tokenizer {
}

private def parseScalarValue(): Token = {
val sb = new StringBuilder
val sb = new StringBuilder
val scalarIndent = in.column

def readScalar(): String =
in.peek() match
case Some(':')
if in.peekNext() == Some(' ') || in.peekNext() == Some('\n') || in
.peekNext() == Some('\r') =>
sb.result()
case Some(':') if in.isNextWhitespace => sb.result()
case Some(char) if !ctx.isAllowedSpecialCharacter(char) => sb.result()
case Some(' ') if in.peekNext() == Some('#') => sb.result()
case Some('\n') | Some('\r') | None => sb.result()
case _ if in.isNewline =>
skipUntilNextChar()
sb.append(' ')
if (in.column > ctx.getIndentOfLatestCollection().getOrElse(Int.MaxValue)) readScalar()
lwronski marked this conversation as resolved.
Show resolved Hide resolved
else sb.result()
case Some(char) =>
sb.append(in.read())
readScalar()
case None => sb.result()

val pos = in.pos()
Scalar(readScalar().trim, ScalarStyle.Plain, pos)
val pos = in.pos()
val scalar = readScalar()
Scalar(scalar.trim, ScalarStyle.Plain, pos)
}

private def fetchValue(): List[Token] =
Expand All @@ -264,35 +265,29 @@ private[yaml] class Scanner(str: String) extends Tokenizer {

in.peek() match
case Some(':') =>
ctx.closeOpenedCollectionMapping(indent)
ctx.closeOpenedCollectionMapping(scalar.pos.column)
in.skipCharacter()

if (ctx.shouldParseMappingEntry(indent)) then
if (ctx.shouldParseMappingEntry(scalar.pos.column)) then
List(Token.Key(scalar.pos), scalar, Token.Value(scalar.pos))
else if (!ctx.isFlowMapping()) then
ctx.appendState(ReaderState.Mapping(indent))
ctx.appendState(ReaderState.Mapping(scalar.pos.column))
List(MappingStart(scalar.pos), Token.Key(scalar.pos), scalar, Token.Value(scalar.pos))
else List(scalar)
case _ => List(scalar)

def skipUntilNextToken(): Unit =
while (in.peek() == Some(' ')) do
indent += 1
in.skipCharacter()
while (in.peek() == Some(' ')) do in.skipCharacter()

if in.peek() == Some('#') then skipComment()

if (in.isNewline) then {
in.skipCharacter()
indent = 0
skipUntilNextToken()
}

def skipUntilNextIndent(indentBlock: Int): Unit =
indent = 0
while (in.peek() == Some(' ') && indent < indentBlock) do
indent += 1
in.skipCharacter()
while (in.peek() == Some(' ') && in.column < indentBlock) do in.skipCharacter()

def skipUntilNextChar() =
while (in.isWhitespace) do in.skipCharacter()
Expand Down
55 changes: 55 additions & 0 deletions yaml/shared/src/test/scala/org/virtuslab/yaml/internal/load/parse/ScalarSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,61 @@ class ScalarSpec extends BaseParseSuite:
assertEventsEquals(events, expectedEvents)
}

test("should parse plain scalar wihth new lines") {
val yaml =
s"""description: new lines
| rest.
|properties: object
|""".stripMargin

val reader = Scanner(yaml)
val events = ParserImpl.getEvents(reader)

val expectedEvents = List(
StreamStart,
DocumentStart(),
MappingStart(),
Scalar("description", ScalarStyle.Plain),
Scalar(
"new lines rest.",
ScalarStyle.Plain
),
Scalar("properties", ScalarStyle.Plain),
Scalar("object", ScalarStyle.Plain),
MappingEnd(),
DocumentEnd(),
StreamEnd
)

assertEventsEquals(events, expectedEvents)
}

test("should parse multine line plain scalar value") {
val yaml =
s"""|description: multiline
| plain
| scalar
|type: string
|""".stripMargin

val reader = Scanner(yaml)
val events = ParserImpl.getEvents(reader)

val expectedEvents = List(
StreamStart,
DocumentStart(),
MappingStart(),
Scalar("description", ScalarStyle.Plain),
Scalar("multiline plain scalar", ScalarStyle.Plain),
Scalar("type", ScalarStyle.Plain),
Scalar("string", ScalarStyle.Plain),
MappingEnd(),
DocumentEnd(),
StreamEnd
)
assertEventsEquals(events, expectedEvents)
}

test("should parse single quote scalar value with multiline") {
val yaml =
s"""description: 'multiline
Expand Down