Tidy up lexer; merge ScalaLexer and Lexer, pull out ModeStack trait, …

…misc

perf/src/main/scala/scalariform/perf/LexerPerformanceTest.scala

@@ -14,21 +14,45 @@ object LexerPerformanceTest {
     val file = new File("/home/matt/coding/scala/src/compiler/scala/tools/nsc/typechecker/Typers.scala")
     val source = Source.fromFile(file).mkString
     println("Source: " + source.length + " chars")
-    1 to 1000 foreach { _ ⇒ doIt(source) }
+    val tokens = ScalaLexer.rawTokenise(source)
+    println("Tokens: " + tokens.size)
+    val ITERATIONS = 3000
+    val WARMUP = 500
+    1 to WARMUP foreach { _ ⇒ doIt(source) }
 
-    val its = 10000
-
     val start = System.currentTimeMillis
-      1 to its foreach { _ => doIt(source) }
+    val durations = 1 to ITERATIONS map { _ =>
+      val start1 = System.nanoTime
+      doIt(source)
+      val duration = System.nanoTime - start1
+      duration.toDouble / 1000000.0
+    }
     val duration = System.currentTimeMillis - start
-    println(duration.toDouble / its + " ms")
-
+    val meanT = duration.toDouble / ITERATIONS
+    println("Raw average: " + meanT)
+    def compute(iterable: Iterable[Double]): (Double, Double) = {
+      def square(x: Double) = x * x
+      val size = iterable.size
+      val mean = durations.sum / size
+      (mean, math.sqrt(durations.map(n => square(n - mean)).sum / size))
+    }
+    val (mean, stdDev) = compute(durations)
+    def isNormal(d: Double) =  math.abs(d - mean) < 2 * stdDev
+    val durations2 = durations.filter(isNormal)
+    println("Original trials: " + ITERATIONS)
+    println("Outliers removed: " + (ITERATIONS - durations2.size))
+    val (mean2, stdDev2) = compute(durations2)
+    println("Minimum: " + durations.min)
+    println("Average: " + mean2 + " ms")
+    println("Standard deviation: " + stdDev2 + " ms")
+    println()
+    durations.foreach(println)
   }
 
-  private def unicodeEscapeReader(s: String) = new UnicodeEscapeReader(s)
-
   private def doIt(s: String) = {
-    UnicodeEscapeDecoder.decode(s)
+    //    UnicodeEscapeDecoder.decode(s)
+    ScalaLexer.rawTokenise(s)
   }
+
 }
 

scalariform/src/main/scala/scalariform/lexer/Keywords.scala

@@ -64,5 +64,5 @@ object Keywords {
     "|" -> PIPE,
     "~" -> TILDE,
     "!" -> EXCLAMATION)
-
-}
+    
+}

scalariform/src/main/scala/scalariform/lexer/LexerMode.scala

@@ -32,5 +32,4 @@ class XmlMode extends LexerMode {
 
   def nestingLevel = tagNestLevel
 
-}
-
+}

scalariform/src/main/scala/scalariform/lexer/ModeStack.scala

@@ -0,0 +1,34 @@
+package scalariform.lexer
+
+import scala.collection.mutable.Stack
+
+/**
+ * Keeping track of nesting level of XML within Scala.
+ */
+trait ModeStack { self: ScalaLexer ⇒
+
+  private val modeStack = new Stack[LexerMode]
+
+  modeStack.push(new ScalaMode)
+
+  protected def popMode() { modeStack.pop() }
+
+  protected def isRootMode = modeStack.size == 1
+
+  protected def switchToScalaModeAndFetchToken() {
+    modeStack.push(new ScalaMode)
+    fetchScalaToken()
+  }
+
+  protected def switchToXmlModeAndFetchToken() {
+    modeStack.push(new XmlMode)
+    fetchXmlToken()
+  }
+
+  protected def xmlMode: XmlMode = modeStack.head.asInstanceOf[XmlMode]
+
+  protected def isXmlMode = modeStack.head.isInstanceOf[XmlMode]
+
+  protected def scalaMode: ScalaMode = modeStack.head.asInstanceOf[ScalaMode]
+
+}