diff --git a/src/main/scala/com/codecommit/gll/Parsers.scala b/src/main/scala/com/codecommit/gll/Parsers.scala index f1c6e57..72dc93d 100644 --- a/src/main/scala/com/codecommit/gll/Parsers.scala +++ b/src/main/scala/com/codecommit/gll/Parsers.scala @@ -21,8 +21,13 @@ trait Parsers { def rep1[A](p: Parser[A]) = p+ - protected def processTail(tail: LineStream) = if (tail.isEmpty) Some(tail) else None - + private def processTail(tail: LineStream) = { + val preProcessed = preProcess(tail) + if (preProcessed.isEmpty) Some(preProcessed) else None + } + + protected def preProcess(s: LineStream) = s + private def canonicalize(str: String) = str.foldLeft("") { (back, c) => val tack = c match { case '\n' => "\\n" @@ -325,7 +330,7 @@ trait Parsers { trait TerminalParser[+R] extends Parser[R] { self => final val terminal = true - final def apply(in: LineStream) = Stream(parse(in) match { + final def apply(in: LineStream) = Stream(parse(preProcess(in)) match { case Success(res, tail) => processTail(tail) match { case Some(tail) => Success(res, tail) case None => Failure(UnexpectedTrailingChars(canonicalize(tail.mkString)), tail) @@ -338,7 +343,7 @@ trait Parsers { * For terminal parsing, this just delegates back to apply() */ def chain(t: Trampoline, in: LineStream)(f: Result[R] => Unit) { - f(parse(in)) + f(parse(preProcess(in))) } protected[gll] def parse(in: LineStream): Result[R] @@ -366,8 +371,8 @@ trait Parsers { } } - def parse(in: LineStream) = self.parse(in) match { - case Success(res1, tail) => other.parse(tail) match { + def parse(in: LineStream) = self.parse(preProcess(in)) match { + case Success(res1, tail) => other.parse(preProcess(tail)) match { case Success(res2, tail) => Success(new ~(res1, res2), tail) case f: Failure => f } @@ -398,9 +403,12 @@ trait Parsers { } def mapWithTail[R2](f: (LineStream, R) => R2): Parser[R2] = new MappedParser[R, R2](self, f) with TerminalParser[R2] { - def parse(in: LineStream) = self.parse(in) match { - case Success(res, tail) => Success(f(in, res), tail) - case x: Failure => x + def parse(in: LineStream) = { + val preProcessed = preProcess(in) + self.parse(preProcessed) match { + case Success(res, tail) => Success(f(preProcessed, res), tail) + case x: Failure => x + } } } } diff --git a/src/main/scala/com/codecommit/gll/RegexParsers.scala b/src/main/scala/com/codecommit/gll/RegexParsers.scala index 5f5ca55..7c45042 100644 --- a/src/main/scala/com/codecommit/gll/RegexParsers.scala +++ b/src/main/scala/com/codecommit/gll/RegexParsers.scala @@ -17,9 +17,6 @@ trait RegexParsers extends Parsers { val wsFirst = if (skipWhitespace) RegexUtils.first(whitespace) else Set[Option[Char]]() Some((wsFirst - None) ++ (super.computeFirst(seen) getOrElse Set[Option[Char]]())) } - - // there should be a way to do this with traits, but I haven't found it yet - override def parse(s: LineStream) = super.parse(handleWhitespace(s)) } } else super.literal(str) } @@ -27,7 +24,7 @@ trait RegexParsers extends Parsers { implicit def regex(r: Regex): RegexParser = { if (skipWhitespace) { new RegexParser(r) { - override def parse(s: LineStream) = super.parse(handleWhitespace(s)) + override def parse(s: LineStream) = super.parse(s) } } else new RegexParser(r) } @@ -40,15 +37,6 @@ trait RegexParsers extends Parsers { implicit def funRegexSyntax(p: Regex) = new RichSyntax1(regex(p)) - override protected def processTail(tail: LineStream) = { - val newTail = if (skipWhitespace) - handleWhitespace(tail) - else - tail - - super.processTail(newTail) - } - private def escapeRegex(str: String) = { val specialChars = Set('[', ']', '{', '}', '\\', '|', '*', '+', '?', '^', '$', '(', ')') @@ -59,8 +47,8 @@ trait RegexParsers extends Parsers { str + c } } - - private def handleWhitespace(s: LineStream) = + + override protected def preProcess(s: LineStream) = s.drop(whitespace findPrefixOf s map { _.length } getOrElse 0) diff --git a/src/test/scala/RegexSpecs.scala b/src/test/scala/RegexSpecs.scala index 91fc8dc..fc21e11 100644 --- a/src/test/scala/RegexSpecs.scala +++ b/src/test/scala/RegexSpecs.scala @@ -77,6 +77,17 @@ object RegexSpecs extends Specification with ScalaCheck with RegexParsers { } } } + + "produce a location of after leading whitespace" in { + case class A(loc: LineStream, x: String) + + val p = literal("daniel") ^# { (loc, x) => A(loc, x) } + + p(" daniel") must beLike { + case Success(A(l, "daniel"), LineStream()) #:: SNil => l.colNum == 5 && l.toString == "daniel" + case _ => false + } + } "eat leading whitespace" in { val p = literal("daniel")