Permalink
Browse files

fix choice combinator, specialize not to only accept matcher

  • Loading branch information...
1 parent f6ecd4a commit bc5a2ee12389dd1a378f1010e33034f69a57544e @freels committed Jul 19, 2011
View
14 src/main/scala/com/twitter/finagle/memcached/Protocol.scala
@@ -36,10 +36,10 @@ object ResponseDecoder {
val readValue = {
val readKey = readTo(" ") // map { decodeUTF8String(_) }
- val readFlags = readTo(" ") map { bytes =>
- decodeFlags(decodeDecimalInt(bytes))
+ val readFlags = readTo(" ") into { bytes =>
+ lift(decodeDecimalInt(bytes)) map { decodeFlags(_) }
}
- val readLength = readUntil(" ", "\r\n") map { decodeDecimalInt(_) }
+ val readLength = readUntil(" ", "\r\n") into { b => lift(decodeDecimalInt(b)) }
val readCas = choice(
" " -> (readLine map { cas => Some(decodeDecimalInt(cas)) }),
"\r\n" -> success(None)
@@ -55,7 +55,7 @@ object ResponseDecoder {
}
val readValues = {
- val readRest = repsep(accept("VALUE ") append readValue, not(guard("END\r\n")))
+ val readRest = repsep(accept("VALUE ") append readValue, not("END\r\n"))
readValue flatMap { first =>
readRest map { rest => Values(first :: rest) }
@@ -67,7 +67,7 @@ object ResponseDecoder {
}
val readStats = {
- val readRest = repsep(accept("STAT ") append readStat, not(guard("END\r\n")))
+ val readRest = repsep(accept("STAT ") append readStat, not("END\r\n"))
readStat flatMap { first =>
readRest map { rest => Stats(first :: rest) }
@@ -96,7 +96,9 @@ object ResponseDecoder {
"VERSION " -> readVersion
)
- val readNumber = readLine map { bytes => Number(decodeDecimalInt(bytes)) }
+ val readNumber = readLine into { bytes =>
+ lift(decodeDecimalInt(bytes)) map { Number(_) }
+ }
val parser: Parser[Response] = readResponse or readNumber
}
View
10 src/main/scala/com/twitter/finagle/parser/incremental/BacktrackingParser.scala
@@ -17,23 +17,23 @@ class BacktrackingParser[+Out](inner: Parser[Out], offset: Int) extends Parser[O
// live with the warning.
inner.decode(buffer) match {
case r: Return[Out] => r
- case c: Continue[Out] => {
- if (c.next == inner && buffer.readerIndex == (start + offset)) {
+ case Continue(next) => {
+ if (next == inner && buffer.readerIndex == (start + offset)) {
buffer.readerIndex(start)
Continue(this)
} else {
val newOffset = buffer.readerIndex - start
buffer.readerIndex(start)
- Continue(new BacktrackingParser(c.next, newOffset))
+ Continue(new BacktrackingParser(next, newOffset))
}
}
case e: Fail => {
buffer.readerIndex(start)
e
}
- case e: Error => {
+ case Error(ex) => {
buffer.readerIndex(start)
- Fail(e.ex)
+ Fail(ex)
}
}
}
View
46 src/main/scala/com/twitter/finagle/parser/incremental/Parser.scala
@@ -7,10 +7,10 @@ import com.twitter.finagle.ParseException
// states: continue (wait), return, error
-sealed abstract class ParseResult[+Output]
+sealed trait ParseResult[+Out]
-case class Continue[+T](next: Parser[T]) extends ParseResult[T]
-case class Return[+T](ret: T) extends ParseResult[T]
+case class Continue[+Out](next: Parser[Out]) extends ParseResult[Out]
+case class Return[+Out](ret: Out) extends ParseResult[Out]
case class Fail(ex: ParseException) extends ParseResult[Nothing]
case class Error(ex: ParseException) extends ParseResult[Nothing]
@@ -50,6 +50,12 @@ abstract class Parser[+Out] {
def map[T](f: Out => T): Parser[T] = this into { out => success(f(out)) }
}
+
+class LiftParser[+Out](r: ParseResult[Out]) extends Parser[Out] {
+ def decode(buffer: ChannelBuffer) = r
+}
+
+
sealed abstract class CompoundParser[+Out] extends Parser[Out] {
override def hasNext = true
@@ -64,22 +70,16 @@ sealed abstract class CompoundParser[+Out] extends Parser[Out] {
step(this)
}
- protected[this] def end(r: ParseResult[Out]) = new ConstParser(r)
+ protected[this] def end(r: ParseResult[Out]) = new LiftParser(r)
}
-
-class ConstParser[+Out](r: ParseResult[Out]) extends Parser[Out] {
- def decode(buffer: ChannelBuffer) = r
-}
-
-
class AppendParser[+Out](parser: Parser[_], tail: Parser[Out]) extends CompoundParser[Out] {
override def decodeStep(buffer: ChannelBuffer) = parser.decode(buffer) match {
- case r: Return[_] => tail
- case c: Continue[_] => if (c.next == parser) {
+ case Return(_) => tail
+ case Continue(next) => if (next eq parser) {
end(Continue(this))
} else {
- end(Continue(new AppendParser(c.next, tail)))
+ end(Continue(new AppendParser(next, tail)))
}
case e: Fail => end(e)
case e: Error => end(e)
@@ -93,11 +93,11 @@ class AppendParser[+Out](parser: Parser[_], tail: Parser[Out]) extends CompoundP
class IntoParser[T, +Out](parser: Parser[T], f: T => Parser[Out])
extends CompoundParser[Out] {
override def decodeStep(buffer: ChannelBuffer) = parser.decode(buffer) match {
- case r: Return[T] => f(r.ret)
- case c: Continue[T] => if (c.next == parser) {
+ case Return(r) => f(r)
+ case Continue(next) => if (next eq parser) {
end(Continue(this))
} else {
- end(Continue(new IntoParser(c.next, f)))
+ end(Continue(new IntoParser(next, f)))
}
case e: Fail => end(e)
case e: Error => end(e)
@@ -113,17 +113,17 @@ extends CompoundParser[Out] {
val start = buffer.readerIndex
choice.decode(buffer) match {
- case r: Return[Out] => end(r)
+ case r: Return[Out] => end(r)
case e: Fail => if (committed || buffer.readerIndex != start) {
end(Error(e.ex))
} else {
tail
}
- case c: Continue[Out] => {
- if (c.next == choice && buffer.readerIndex == start) {
+ case Continue(next) => {
+ if ((next eq choice) && buffer.readerIndex == start) {
end(Continue(this))
} else {
- end(Continue(new OrParser(c.next, tail, committed || buffer.readerIndex != start)))
+ end(Continue(new OrParser(next, tail, committed || buffer.readerIndex != start)))
}
}
case e: Error => end(e)
@@ -155,13 +155,13 @@ class NotParser(parser: Parser[_]) extends Parser[Unit] {
Return(())
}
}
- case c: Continue[_] => {
+ case Continue(next) => {
if (buffer.readerIndex != start) {
error()
- } else if (c.next == parser) {
+ } else if (next == parser) {
Continue(this)
} else {
- Continue(new NotParser(c.next))
+ Continue(new NotParser(next))
}
}
case e: Error => e
View
117 src/main/scala/com/twitter/finagle/parser/incremental/Parsers.scala
@@ -6,69 +6,29 @@ import com.twitter.finagle.ParseException
object Parsers {
- def readTo(choices: String*) = {
- new ConsumingDelimiterParser(AlternateMatcher(choices))
- }
-
- def readUntil(choices: String*) = {
- new DelimiterParser(AlternateMatcher(choices))
- }
-
- val readLine = readTo("\r\n", "\n")
-
- def fail(ex: ParseException) = new ConstParser(Fail(ex))
- def error(ex: ParseException) = new ConstParser(Error(ex))
+ // lifting values
- def success[T](t: T) = new ConstParser(Return(t))
+ def fail(ex: ParseException) = new LiftParser(Fail(ex))
- def lift[T](f: => T): Parser[T] = {
- try {
- success(f)
- } catch {
- case e: ParseException => fail(e)
- }
- }
+ def error(ex: ParseException) = new LiftParser(Error(ex))
- def attempt[T](p: Parser[T]) = new BacktrackingParser(p)
+ def success[T](t: T) = new LiftParser(Return(t))
val unit = success(())
- def readBytes(size: Int) = new FixedBytesParser(size)
-
- def skipBytes(size: Int) = readBytes(size) append unit
-
- def accept(m: Matcher) = new ConsumingMatchParser(m)
-
- implicit def accept(choice: String): Parser[ChannelBuffer] = {
- accept(new DelimiterMatcher(choice))
- }
-
- def accept(choices: String*): Parser[ChannelBuffer] = {
- accept(AlternateMatcher(choices))
+ def lift[T](o: Option[T]): Parser[T] = o match {
+ case Some(r) => success(r)
+ case None => fail(new ParseException("Parse failed."))
}
- def guard(m: Matcher) = new MatchParser(m)
-
- def guard(choice: String): Parser[ChannelBuffer] = {
- guard(new DelimiterMatcher(choice))
- }
- def guard(choices: String*): Parser[ChannelBuffer] = {
- guard(AlternateMatcher(choices))
- }
+ // backtrack
- def not(m: Parser[Any]) = new NotParser(m)
+ def attempt[T](p: Parser[T]) = new BacktrackingParser(p)
- def choice[T](choices: (String, Parser[T])*) = {
- val (m, p) = choices.last
- val last: Parser[T] = accept(m) append p
- (choices.tail.reverse foldRight last) { (choice, rest) =>
- val (m, p) = choice
- (accept(m) append p) or rest
- }
- }
+ // repetition
def rep[T](p: Parser[T]): Parser[List[T]] = {
def go(): Parser[List[T]] = {
@@ -119,6 +79,63 @@ object Parsers {
}
+ // matching parsers
+
+ def accept(m: Matcher) = new ConsumingMatchParser(m)
+
+ implicit def accept(choice: String): Parser[ChannelBuffer] = {
+ accept(new DelimiterMatcher(choice))
+ }
+
+ def accept(choices: String*): Parser[ChannelBuffer] = {
+ accept(AlternateMatcher(choices))
+ }
+
+ def guard(m: Matcher) = new MatchParser(m)
+
+ def guard(choice: String): Parser[ChannelBuffer] = {
+ guard(new DelimiterMatcher(choice))
+ }
+
+ def guard(choices: String*): Parser[ChannelBuffer] = {
+ guard(AlternateMatcher(choices))
+ }
+
+ def not(m: Matcher) = new MatchParser(new NotMatcher(m))
+
+ def not(choice: String): Parser[ChannelBuffer] = {
+ not(new DelimiterMatcher(choice))
+ }
+
+ def not(choices: String*): Parser[ChannelBuffer] = {
+ not(AlternateMatcher(choices))
+ }
+
+
+
+ def choice[T](choices: (String, Parser[T])*): Parser[T] = {
+ val (m, p) = choices.first
+ val first: Parser[T] = accept(m) append p
+ val rest = choices.tail
+
+ if (rest.isEmpty) first else first or choice(rest: _*)
+ }
+
+ def readTo(choices: String*) = {
+ new ConsumingDelimiterParser(AlternateMatcher(choices))
+ }
+
+ def readUntil(choices: String*) = {
+ new DelimiterParser(AlternateMatcher(choices))
+ }
+
+ val readLine = readTo("\r\n", "\n")
+
+
+ // basic reading parsers
+
+ def readBytes(size: Int) = new FixedBytesParser(size)
+
private[parser] abstract class PrimitiveParser[Out] extends Parser[Out] {
protected val continue = Continue(this)
}
View
10 src/main/scala/com/twitter/finagle/parser/util/DecimalIntCodec.scala
@@ -44,11 +44,11 @@ object DecimalIntCodec {
}
}
- def decode(buf: ChannelBuffer): Int = {
+ def decode(buf: ChannelBuffer): Option[Int] = {
decode(buf, buf.readableBytes)
}
- def decode(buf: ChannelBuffer, numBytes: Int): Int = {
+ def decode(buf: ChannelBuffer, numBytes: Int): Option[Int] = {
val last = numBytes - 1
var i = last
var rv = 0
@@ -65,19 +65,19 @@ object DecimalIntCodec {
while (i >= lower) {
val c = buf.getByte(buf.readerIndex + i) - AsciiZero
- if (c < 0 || c > 9) throw new ParseException("byte out of bounds")
+ if (c < 0 || c > 9) return None
rv = rv + c * pow(10, last - i)
i = i - 1
}
- if (isNegative) rv * -1 else rv
+ if (isNegative) Some(rv * -1) else Some(rv)
}
// helpers
private def pow(x: Int, p: Int) = {
var rv = 1
- var j = 0
+ var j = 0
while (j < p) {
rv = rv * x
View
12 src/main/scala/com/twitter/finagle/parser/util/Matcher.scala
@@ -42,6 +42,8 @@ class DelimiterMatcher(delimiter: Array[Byte]) extends Matcher {
delimiter.length
}
+
+ override def toString = "DelimiterMatcher("+ (new String(delimiter, "US-ASCII")) +")"
}
object AlternateMatcher {
@@ -73,3 +75,13 @@ class AlternateMatcher(delimiters: Array[Array[Byte]]) extends Matcher {
-1
}
}
+
+class NotMatcher(inner: Matcher) extends Matcher {
+ val bytesNeeded = inner.bytesNeeded
+
+ def bytesMatching(buffer: ChannelBuffer, offset: Int): Int = {
+ if (buffer.writerIndex < offset + bytesNeeded) return -1
+
+ if (inner.bytesMatching(buffer, offset) == -1) 0 else -1
+ }
+}
View
6 src/main/scala/com/twitter/finagle/redis/protocol/Reply.scala
@@ -19,7 +19,7 @@ object ReplyDecoder {
import Reply._
import Parsers._
- private val readDecimalInt = readLine into { bytes => lift(decodeDecimalInt(bytes)) }
+ private val readDecimalInt = readLine into { b => lift(decodeDecimalInt(b)) }
private val readStatusReply = readLine map { Status(_) }
@@ -32,9 +32,7 @@ object ReplyDecoder {
success(Bulk(None))
} else {
readBytes(size) flatMap { bytes =>
- skipBytes(2) map { _ =>
- Bulk(Some(bytes))
- }
+ readBytes(2) append success(Bulk(Some(bytes)))
}
}
}
View
60 src/test/scala/com/twitter/finagle/parser/IncrementalParserSpec.scala
@@ -97,11 +97,6 @@ object ParserSpec extends ParserSpecification {
readBytes(7) map asString mustParse "aaaaaa" andContinue()
}
- "skipBytes" in {
- skipBytes(2) mustParse "abc" andReturn () readingBytes(2)
- skipBytes(4) mustParse "abc" andContinue()
- }
-
"accept" in {
val parser = Parsers.accept("$") append (readBytes(1) map asString)
@@ -232,37 +227,38 @@ object ParserSpec extends ParserSpecification {
}
}
- def time[T](f: => T) = {
- val s = System.currentTimeMillis
- f
- val e = System.currentTimeMillis
- e - s
- }
- "performance" in {
- val readInt = readLine map { bytes => decodeDecimalInt(bytes) }
- val readBulk = Parsers.accept("$") append (readInt into { length =>
- readBytes(length) into { bytes =>
- readBytes(2) append success(bytes)
- }
- })
+ // "performance" in {
+ // def time[T](f: => T) = {
+ // val s = System.currentTimeMillis
+ // f
+ // val e = System.currentTimeMillis
+ // e - s
+ // }
- val test1 = Parsers.accept("*") append (readInt into { count =>
- repN(count, readBulk)
- })
+ // val readInt = readLine map { bytes => decodeDecimalInt(bytes) }
+ // val readBulk = Parsers.accept("$") append (readInt into { length =>
+ // readBytes(length) into { bytes =>
+ // readBytes(2) append success(bytes)
+ // }
+ // })
- val count = 100
- val buf1 = ChannelBuffers.wrappedBuffer(("*"+count+"\r\n" + ("$6\r\nfoobar\r\n" * count)).getBytes)
+ // val test1 = Parsers.accept("*") append (readInt into { count =>
+ // repN(count, readBulk)
+ // })
- println(test1.decode(buf1))
+ // val count = 100
+ // val buf1 = ChannelBuffers.wrappedBuffer(("*"+count+"\r\n" + ("$6\r\nfoobar\r\n" * count)).getBytes)
- for (x <- 1 to 100) {
- val rv = time { for (i <- 1 to 100000) {
- buf1.resetReaderIndex
- test1.decode(buf1)
- } }
+ // println(test1.decode(buf1))
- println("test 1: "+ rv +" ("+ (rv / 100000.0) +")")
- }
- }
+ // for (x <- 1 to 100) {
+ // val rv = time { for (i <- 1 to 100000) {
+ // buf1.resetReaderIndex
+ // test1.decode(buf1)
+ // } }
+
+ // println("test 1: "+ rv +" ("+ (rv / 100000.0) +")")
+ // }
+ // }
}

0 comments on commit bc5a2ee

Please sign in to comment.