diff --git a/README.md b/README.md index c1e3630..d13e58f 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,20 @@ scalaDeps = [ ] ``` +#### Language Extensions +toml-scala supports the following language extensions which are disabled by default: + +* [New lines and trailing commas in inline tables](https://github.com/toml-lang/toml/issues/516) + +To enable them, pass in a set of extensions to the `parse()` or `parseAs()` function as a second argument: + +```scala +toml.Toml.parse("""key = { + a = 23, + b = 42, +}""", Set(toml.Extension.MultiLineInlineTables)) +``` + ## Links * [ScalaDoc](https://www.javadoc.io/doc/tech.sparse/toml-scala_2.12/) diff --git a/jvm/src/main/scala/toml/PlatformRules.scala b/jvm/src/main/scala/toml/PlatformRules.scala index 3f05411..94bfb12 100644 --- a/jvm/src/main/scala/toml/PlatformRules.scala +++ b/jvm/src/main/scala/toml/PlatformRules.scala @@ -4,7 +4,7 @@ import java.time._ import scala.meta.internal.fastparse.all._ -trait PlatformRules { this: Rules.type => +trait PlatformRules { this: Rules => private val TenPowers = List(1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000) @@ -39,4 +39,4 @@ trait PlatformRules { this: Rules.type => } val date = P(offsetDateTime | localDateTime | localDate | localTime) -} \ No newline at end of file +} diff --git a/shared/src/main/scala/toml/Rules.scala b/shared/src/main/scala/toml/Rules.scala index 50edf00..3f03391 100644 --- a/shared/src/main/scala/toml/Rules.scala +++ b/shared/src/main/scala/toml/Rules.scala @@ -9,26 +9,31 @@ private[toml] case class NamedFunction[T, V](f: T => V, name: String) override def toString: String = name } -object Rules extends PlatformRules { +sealed trait Extension +object Extension { + case object MultiLineInlineTables extends Extension +} + +case object Rules extends toml.Rules(Set()) + +class Rules(extensions: Set[Extension]) extends PlatformRules { import Constants._ + import Extension._ val UntilNewline = NamedFunction(!CrLf.contains(_: Char), "UntilNewline") val newLine = P(StringIn(CrLf, Lf)) val charsChunk = P(CharsWhile(UntilNewline)) val comment = P("#" ~ charsChunk.? ~ &(newLine | End)) + val whitespace = P(CharIn(WhitespaceChars.toList)) - val whitespace = P(CharIn(WhitespaceChars.toList)) - val whitespaces = P(whitespace.rep(1)) + val skip = P(NoCut(NoTrace((whitespace | comment | newLine).rep))) + val skipWs = P(NoCut(NoTrace(whitespace.rep))) - val skip = P(NoCut(NoTrace((whitespaces | comment | newLine).rep))) - - val letter = P(CharIn(LettersRange)) - val letters = P(letter.rep(1)) - val digit = P(CharIn(NumbersRange)) - val digits = P(digit.rep(1)) - - val skipSpaces = P(CharsWhile(_.isWhitespace).?) + val letter = P(CharIn(LettersRange)) + val digit = P(CharIn(NumbersRange)) + val digits = P(digit.rep(1)) + val dash = P(CharIn(Dashes.toList)) val StringChars = NamedFunction(!"\"\\".contains(_: Char), "StringChars") val strChars = P(CharsWhile(StringChars)) @@ -52,14 +57,14 @@ object Rules extends PlatformRules { val multiLineBasicStr: Parser[Value.Str] = P( MultiLineDoubleQuote ~/ - skipSpaces ~ + newLine.? ~ (!MultiLineDoubleQuote ~ AnyChar).rep.! ~ MultiLineDoubleQuote ).map(str => Value.Str(Unescape.unescapeJavaString(str))) val multiLineLiteralStr: Parser[Value.Str] = P( MultiLineSingleQuote ~/ - skipSpaces ~ + newLine.? ~ (!MultiLineSingleQuote ~ AnyChar).rep.! ~ MultiLineSingleQuote ).map(Value.Str) @@ -98,44 +103,42 @@ object Rules extends PlatformRules { val `false` = P("false").map(_ => Value.Bool(false)) val boolean = P(`true` | `false`) - val dashes = P(CharIn(Dashes.toList)) - val bareKey = P((letters | digits | dashes).rep(min = 1)).! + val bareKey = P((letter | digit | dash).rep(min = 1)).! val validKey: Parser[String] = P(NoCut(basicStr.map(_.value)) | NoCut(literalStr.map(_.value)) | bareKey) val pair: Parser[(String, Value)] = - P(validKey ~ whitespaces.? ~ "=" ~ whitespaces.? ~ elem) + P(validKey ~ skipWs ~ "=" ~ skipWs ~ elem) val array: Parser[Value.Arr] = P("[" ~ skip ~ elem.rep(sep = "," ~ skip) ~ ",".? ~ skip ~ "]") .map(l => Value.Arr(l.toList)) val inlineTable: Parser[Value.Tbl] = - P("{" ~ skip ~ pair.rep(sep = "," ~ skip) ~ ",".? ~ skip ~ "}") - .map(p => Value.Tbl(p.toMap)) + (if (extensions.contains(MultiLineInlineTables)) + P("{" ~ skip ~ pair.rep(sep = "," ~ skip) ~ ",".? ~ skip ~ "}") + else + P("{" ~ skipWs ~ pair.rep(sep = "," ~ skipWs) ~ skipWs ~ "}") + ).map(p => Value.Tbl(p.toMap)) val tableIds: Parser[Seq[String]] = - P(validKey.rep(min = 1, sep = whitespaces.? ~ "." ~ whitespaces.?).map(_.toSeq)) + P(validKey.rep(min = 1, sep = skipWs ~ "." ~ skipWs).map(_.toSeq)) val tableDef: Parser[Seq[String]] = - P("[" ~ whitespaces.? ~ tableIds ~ whitespaces.? ~ "]") + P("[" ~ skipWs ~ tableIds ~ skipWs ~ "]") val tableArrayDef: Parser[Seq[String]] = - P("[[" ~ whitespaces.? ~ tableIds ~ whitespaces.? ~ "]]") + P("[[" ~ skipWs ~ tableIds ~ skipWs ~ "]]") val pairNode: Parser[Node.Pair] = pair.map { case (k, v) => Node.Pair(k, v) } val table: Parser[Node.NamedTable] = - P(skip ~ tableDef ~ skip ~ pair.rep(sep = skip)).map { case (a, b) => + P(tableDef ~ skip ~ pair.rep(sep = skip)).map { case (a, b) => Node.NamedTable(a.toList, b.toList) } val tableArray: Parser[Node.NamedArray] = - P(skip ~ tableArrayDef ~ skip ~ pair.rep(sep = skip)).map { case (a, b) => + P(tableArrayDef ~ skip ~ pair.rep(sep = skip)).map { case (a, b) => Node.NamedArray(a.toList, b.toList) } - lazy val elem: Parser[Value] = P { - skip ~ - (date | string | boolean | double | integer | array | inlineTable) ~ - skip - } - - lazy val node: Parser[Node] = P(skip ~ (pairNode | table | tableArray)) + lazy val elem: Parser[Value] = + P(date | string | boolean | double | integer | array | inlineTable) - val root: Parser[Root] = P(node.rep(sep = skip) ~ skip ~ End) + val node: Parser[Node] = P(pairNode | table | tableArray) + val root: Parser[Root] = P(skip ~ node.rep(sep = skip) ~ skip ~ End) .map(nodes => Root(nodes.toList)) } diff --git a/shared/src/main/scala/toml/Toml.scala b/shared/src/main/scala/toml/Toml.scala index 38f0d66..751ca62 100644 --- a/shared/src/main/scala/toml/Toml.scala +++ b/shared/src/main/scala/toml/Toml.scala @@ -5,8 +5,8 @@ import shapeless._ import scala.meta.internal.fastparse.core.Parsed._ object Toml { - def parse(toml: String): Either[Parse.Error, Value.Tbl] = - Rules.root.parse(toml) match { + def parse(toml: String, extensions: Set[Extension] = Set()): Either[Parse.Error, Value.Tbl] = + new Rules(extensions).root.parse(toml) match { case Success(v, _) => Embed.root(v) case f: Failure[_, _] => Left(List() -> f.msg) } @@ -24,23 +24,37 @@ object Toml { codec(table, d, 0).right.map(generic.from) } - def apply[D <: HList, R <: HList](toml: String)(implicit + def apply[D <: HList, R <: HList]( + toml : String, + extensions: Set[Extension] + )(implicit generic : LabelledGeneric.Aux[A, R], defaults : Default.AsRecord.Aux[A, D], defaultMapper: util.RecordToMap[D], codec : Codec[R] ): Either[Parse.Error, A] = { val d = defaultMapper(defaults()) - parse(toml).right.flatMap(codec(_, d, 0).right.map(generic.from)) + parse(toml, extensions) + .right + .flatMap(codec(_, d, 0).right.map(generic.from)) } + + def apply[D <: HList, R <: HList](toml: String)( + implicit + generic : LabelledGeneric.Aux[A, R], + defaults : Default.AsRecord.Aux[A, D], + defaultMapper: util.RecordToMap[D], + codec : Codec[R] + ): Either[Parse.Error, A] = apply(toml, Set()) } class CodecHelperValue[A] { def apply(value: Value)(implicit codec: Codec[A]): Either[Parse.Error, A] = codec(value, Map(), 0) - def apply(toml: String)(implicit codec: Codec[A]): Either[Parse.Error, A] = - parse(toml).right.flatMap(codec(_, Map(), 0)) + def apply(toml: String, extensions: Set[Extension] = Set()) + (implicit codec: Codec[A]): Either[Parse.Error, A] = + parse(toml, extensions).right.flatMap(codec(_, Map(), 0)) } def parseAs [T]: CodecHelperGeneric[T] = new CodecHelperGeneric[T] diff --git a/shared/src/test/scala/toml/CodecSpec.scala b/shared/src/test/scala/toml/CodecSpec.scala index 42318ab..1d2fe94 100644 --- a/shared/src/test/scala/toml/CodecSpec.scala +++ b/shared/src/test/scala/toml/CodecSpec.scala @@ -62,7 +62,7 @@ class CodecSpec extends FunSuite { case class Pair(a: Int) val pair = """a = 1""" - assert(Toml.parseAs[Pair](pair) == Right(Pair(1))) + assert(Toml.parseAs[Pair](pair, Set()) == Right(Pair(1))) case class Pairs(a: Int, b: Int) val pairs = diff --git a/shared/src/test/scala/toml/GeneratedSpec.scala b/shared/src/test/scala/toml/GeneratedSpec.scala index 594253f..2d62ccf 100644 --- a/shared/src/test/scala/toml/GeneratedSpec.scala +++ b/shared/src/test/scala/toml/GeneratedSpec.scala @@ -63,10 +63,10 @@ class GeneratedSpec extends PropSpec with PropertyChecks with Matchers { } } - property("Parse pairs (with `node` parser)") { + property("Parse pairs (with `root` parser)") { import Generators.Tables._ forAll(pairGen) { s: String => - shouldBeSuccess(Rules.node.parse(s)) + shouldBeSuccess(Rules.root.parse(s)) } } @@ -80,14 +80,14 @@ class GeneratedSpec extends PropSpec with PropertyChecks with Matchers { property("Parse tables") { import Generators.Tables._ forAll(tableGen) { s: String => - shouldBeSuccess[Node.NamedTable](Rules.table.parse(s)) + shouldBeSuccess[Node.NamedTable]((Rules.skip ~ Rules.table).parse(s)) } } - property("Parse tables (with `node` parser)") { + property("Parse tables (with `root` parser)") { import Generators.Tables._ forAll(tableGen) { s: String => - shouldBeSuccess(Rules.node.parse(s)) + shouldBeSuccess(Rules.root.parse(s)) } } } diff --git a/shared/src/test/scala/toml/ParseSpec.scala b/shared/src/test/scala/toml/ParseSpec.scala index 6ca88fb..92fdbc5 100644 --- a/shared/src/test/scala/toml/ParseSpec.scala +++ b/shared/src/test/scala/toml/ParseSpec.scala @@ -6,6 +6,27 @@ import Node._ import org.scalatest.FunSuite class ParseSpec extends FunSuite { + test("Parse strings") { + val toml = + """ + |lines = ''' + | + |The first newline is + |trimmed in raw strings. + | All other whitespace + | is preserved. + |''' + |""".stripMargin + + val result = Toml.parse(toml) + assert(result == Right(Tbl(Map("lines" -> Value.Str( + "\n" + + "The first newline is\n" + + "trimmed in raw strings.\n" + + " All other whitespace\n" + + " is preserved.\n"))))) + } + test("Redefine value on root level") { val toml = """a = 23 @@ -34,4 +55,14 @@ class ParseSpec extends FunSuite { val result = Toml.parse(toml) assert(result == Left((List("a", "b"), "Cannot redefine value"))) } + + test("Extension: Parse inline tables with trailing comma") { + val result = Toml.parse("""key = { + a = 23, + b = 42, + }""", Set(Extension.MultiLineInlineTables)) + + assert(result == Right(Tbl( + Map("key" -> Tbl(Map("a" -> Num(23), "b" -> Num(42))))))) + } } diff --git a/shared/src/test/scala/toml/RulesSpec.scala b/shared/src/test/scala/toml/RulesSpec.scala index 8ee44c6..1ab3b06 100644 --- a/shared/src/test/scala/toml/RulesSpec.scala +++ b/shared/src/test/scala/toml/RulesSpec.scala @@ -63,7 +63,41 @@ class RulesSpec extends FunSuite with Matchers { testSuccess(example) } - test("Parse inline tables") { + test("Parse inline tables (1)") { + val example = + """ + |a = { name = "value", name2 = "value2" } + |b = { name = "value" } + """.stripMargin + + testSuccess(example) + } + + test("Extension: Parse inline tables with new line") { + // See https://github.com/toml-lang/toml/issues/516 + val example = + """ + |a = { name = "value", name2 = "value2" } + |b = { name = "value" + | } + """.stripMargin + + testFailure(example) + testSuccess(example, new Rules(Set(Extension.MultiLineInlineTables))) + } + + test("Extension: Parse inline tables with trailing comma") { + val example = + """ + |a = { name = "value", name2 = "value2" } + |b = { name = "value", } + """.stripMargin + + testFailure(example) + testSuccess(example, new Rules(Set(Extension.MultiLineInlineTables))) + } + + test("Extension: Parse inline tables with trailing comma and comment") { val example = """ |a = { name = "value", name2 = "value2" } @@ -71,7 +105,9 @@ class RulesSpec extends FunSuite with Matchers { | # Trailing comma | } """.stripMargin - testSuccess(example) + + testFailure(example) + testSuccess(example, new Rules(Set(Extension.MultiLineInlineTables))) } test("Parse complex table keys") { diff --git a/shared/src/test/scala/toml/TestHelpers.scala b/shared/src/test/scala/toml/TestHelpers.scala index 7f7e218..a35ca2e 100644 --- a/shared/src/test/scala/toml/TestHelpers.scala +++ b/shared/src/test/scala/toml/TestHelpers.scala @@ -8,14 +8,14 @@ import org.scalatest.Matchers object TestHelpers { import Matchers._ - def testSuccess(example: String): Root = - Rules.root.parse(example) match { + def testSuccess(example: String, rules: Rules = Rules): Root = + rules.root.parse(example) match { case Success(v, _) => v case f: Failure[_, _] => fail(s"Failed to parse `$example`: ${f.msg}") } - def testFailure(example: String): Unit = - Rules.root.parse(example) match { + def testFailure(example: String, rules: Rules = Rules): Unit = + rules.root.parse(example) match { case Success(_, _) => fail(s"Did not fail: $example") case _: Failure[_, _] => } @@ -29,4 +29,4 @@ object TestHelpers { case s: Success[T, _, _] => fail(s"$r is not a Failure.") case f: Failure[_, _] => } -} \ No newline at end of file +}