Skip to content

Commit

Permalink
Add fixes to parser, user API and hard_toml test case
Browse files Browse the repository at this point in the history
This commit fixes several issues in the parser, especially for trailing commas and double quotes within escaped strings. It also adds a de-facto unit test to check that the parser is valid.

Aside from that, user API has been improved, especially the conversion from string to key, and a new method `filter` has been added to make the collection of certain key-value pairs easier.
  • Loading branch information
jvican committed Oct 14, 2016
1 parent aaf2e9f commit 0a1cf30
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 44 deletions.
13 changes: 9 additions & 4 deletions src/main/scala/stoml/TomlParser.scala
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ trait TomlParser extends ParserUtil with TomlSymbol {
lazy val pair: Parser[Pair] =
P(validKey ~ WS0.? ~ "=" ~ WS0.? ~ elem) map Pair
lazy val array: Parser[Arr] =
P("[" ~ WS ~ elem.rep(sep = WS0.? ~ "," ~/ WS) ~ WS ~ "]") map Arr
P("[" ~ WS ~ elem.rep(sep = "," ~ WS) ~ ",".? ~ WS ~ "]") map Arr

val tableIds: Parser[Seq[String]] =
P(validKey.rep(min = 1, sep = WS0.? ~ "." ~ WS0.?))
Expand All @@ -189,10 +189,15 @@ trait TomlParserApi extends TomlParser {
import stoml.Toml.{Node, Table, Pair}

type Key = Vector[String]
implicit def stringToKey(key: String): Key = key.split(".").toVector
implicit def stringToKey(key: String): Key = {
require(!key.isEmpty, "Key must be non-empty")
key.split(".").toVector
}

case class TomlContent(private val c: Map[Key, Node]) {
def lookup(k: Key): Option[Node] = c.get(k)
case class TomlContent(map: Map[Key, Node]) {
def lookup(k: Key): Option[Node] = map.get(k)
def filter(f: Key => Boolean): Iterator[Node] =
map.filterKeys(f).values.iterator
}

object TomlContent {
Expand Down
31 changes: 0 additions & 31 deletions src/test/scala/stoml/HardToml.scala

This file was deleted.

92 changes: 92 additions & 0 deletions src/test/scala/stoml/HardTomlSpec.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package stoml

import java.io.File

import org.scalatest.{FlatSpec, Matchers}
import fastparse.core.Parsed.{Failure, Success}

class HardTomlSpec extends FlatSpec with Matchers {

import stoml.TomlParserApi._

def testParser(example: String) = {
val parsed = parseToml(example)
parsed match {
case Success(v, _) =>
case f: Failure =>
fail(s"Failed to parse `$example`: $f")
}
}

def testFailingParser(example: String) = {
val parsed = parseToml(example)
parsed match {
case Success(v, _) =>
fail(s"Didn't fail to parse `$example`.")
case f: Failure =>
}
}

"The TOML parser" should "be able to parse the hard TOML example" in {
val parsed = parseToml(new File("./src/test/scala/stoml/hard_example.toml"))
parsed match {
case Success(v, _) =>
case f: Failure =>
fail(s"The hard example failed with: $f")
}
}

it should "parse escaped double quotes inside a string" in {
val example =
"""
|harder_test_string = " And when \"'s are in the string, along with # \"" # "and comments are there too"
""".stripMargin
testParser(example)
}

it should "parse complex table keys" in {
val example =
"""[asdf."bit#"]
|"hello" = "asdfasdf"
""".stripMargin
testParser(example)
}

it should "parse multi-line array with trailing commas" in {
val example =
"""
|multi_line_array = [
| "]",
| # ] Oh yes I did
| ]
""".stripMargin
testParser(example)
}

it should "fail to parse non-toml-compliant statement" in {
val example = "[error] if you didn't catch this, your parser is broken"
testFailingParser(example)
}

it should "fail to parse comment at EOL" in {
val example = "string = \"Anything other than tabs, spaces and newline after a keygroup or key value pair has ended should produce an error unless it is a comment\" like this"
testFailingParser(example)
}

it should "fail to parse end of comment after tricky array declaration" in {
val example =
"""array = [
| "This might most likely happen in multiline arrays",
| Like here,
| "or here,
| and here"
|] End of array comment, forgot the #
""".stripMargin
testFailingParser(example)
}

it should "fail to parse comment at the end of key-pair definition" in {
val example = "number = 3.14 pi <--again forgot the #"
testFailingParser(example)
}
}
10 changes: 1 addition & 9 deletions src/test/scala/stoml/hard_example_error.toml
Original file line number Diff line number Diff line change
@@ -1,11 +1,3 @@
# Each of the following keygroups/key value pairs should produce an error. Uncomment to them to test

#[error] if you didn't catch this, your parser is broken
#string = "Anything other than tabs, spaces and newline after a keygroup or key value pair has ended should produce an error unless it is a comment" like this
#array = [
# "This might most likely happen in multiline arrays",
# Like here,
# "or here,
# and here"
# ] End of array comment, forgot the #
#number = 3.14 pi <--again forgot the #
number = 3.14 pi <--again forgot the #

0 comments on commit 0a1cf30

Please sign in to comment.