Skip to content
Permalink
Browse files
Eliminated DFAStatus structure.
Reduces allocation in inner loop of DFA (aka tokenizer or lexical
analyzer)

Note: There is still lots of per-element allocation. This just
eliminates the per-character allocation of DFAStatus objects.

Also converts from Seq[Delim] to Array so that we can open-code a
foreach loop as a while loop knowing that the indexing into the array is
constant time.

Added additional scala 2.11 compiler flags to force more aggressive
inlining. Things like OnStack were still allocating closures without
these flags.

DFDL-1415
  • Loading branch information
mbeckerle committed Oct 29, 2015
1 parent 89a1601 commit 57b2fce13a97ec7c9ce1485a52998b6aefd4a0c4
Show file tree
Hide file tree
Showing 30 changed files with 432 additions and 350 deletions.
@@ -1,3 +1,5 @@


name := "daffodil"

organization in ThisBuild := "edu.illinois.ncsa"
@@ -7,7 +9,7 @@ scalaVersion in ThisBuild := "2.11.7"
// incOptions := incOptions.value.withNameHashing(true) // 2.11 experimental incremental compilation improvements (perhaps not working right?)

scalacOptions in ThisBuild ++= Seq("-unchecked", "-deprecation", "-Yinline-warnings", "-Xfatal-warnings", "-Xxml:-coalescing", "-language:experimental.macros", "-Ybackend:GenBCode", "-Yopt-warnings",
"-Ywarn-inaccessible", "-Ywarn-unused-import", "-Ywarn-unused", "-Ywarn-infer-any", "-Ywarn-nullary-override", "-Ydead-code" )
"-Ywarn-inaccessible", "-Ywarn-unused-import", "-Ywarn-unused", "-Ywarn-infer-any", "-Ywarn-nullary-override", "-Ydead-code", "-Yopt:inline-global", "-Yinline" )

// parallelExecution in ThisBuild := false

@@ -369,7 +369,8 @@ class TestDFDLParser {
</xs:sequence>
</xs:complexType>
</xs:element>)
val actual = TestUtils.testString(sch, "5;6;7;8;A.").result
val areTracing = false
val actual = TestUtils.testString(sch, "5;6;7;8;A.", areTracing).result
val expected = <e1><s1>5</s1><s1>6</s1><s1>7</s1><s1>8</s1><s2>A</s2></e1>
TestUtils.assertEqualsXMLElements(expected, actual)
}
@@ -671,6 +671,9 @@ trait DataInputStream
fillCharBufferLoop(cb)
if (cb.position() == 0) Nope
else Maybe(cb.flip.toString)
// TODO: Performance - we need to copy here. Consider return type of Maybe[StringBuilder]
// as that will allow for non-copying trim and other manipulations of the string
// without further copyies.
}
}

@@ -74,6 +74,7 @@ object Implicits {
body
None
} catch {
case npe: NullPointerException => throw npe
case s: scala.util.control.ControlThrowable => throw s
case u: Throwable => {
if (!clazz.isAssignableFrom(u.getClass)) {
@@ -505,8 +505,9 @@ object OOLAG extends Logging {

protected final def oolagFinalize = {
setIndent(indent - 2)
// log(LogLevel.OOLAGDebug, " " * indent + "pop: " + thisThing))
oolagContext.currentOVList = oolagContext.currentOVList.tail
log(LogLevel.OOLAGDebug, " " * indent + "pop: " + thisThing)
if (oolagContext.currentOVList.nonEmpty)
oolagContext.currentOVList = oolagContext.currentOVList.tail
}

final def hasError = alreadyTriedThis && !hasValue
@@ -581,6 +582,7 @@ object OOLAG extends Logging {
oolagAfterValue(v.asInstanceOf[AnyRef])
v
} catch {
case npe: NullPointerException => throw npe
case s: scala.util.control.ControlThrowable => throw s
case u: UnsuppressableException => throw u
case e: Error => throw e
@@ -209,8 +209,10 @@ trait Logging extends Identity {
else className
}

private var logWriter: Maybe[LogWriter] = Nope
protected var logLevel: Maybe[LogLevel.Type] = Nope
// Note: below can't be private or protected because macro expansions refer to them,
// and they would be unreachable from the location of the expansion of the macro.
var logWriter: Maybe[LogWriter] = Nope
var logLevel: Maybe[LogLevel.Type] = Nope

def setLoggingLevel(level: LogLevel.Type) { logLevel = One(level) }

@@ -229,9 +231,12 @@ trait Logging extends Identity {
protected def doLogging(lvl: LogLevel.Type, msg: String, args: Seq[Any]) =
getLogWriter.log(lvl, logID, msg, args)

// TODO: Convert this into a macro so we don't end up allocating closures
// for these by-name args.
final def log(lvl: LogLevel.Type, msg: String, args: Any*): Unit = macro LoggerMacros.logMacro
// {
// val l = level.lvl
// if (getLoggingLevel().lvl >= l)
// doLogging(level, msg, args.toSeq)
// }

/**
* Use to make debug printing over small code regions convenient. Turns on
@@ -47,10 +47,10 @@ import edu.illinois.ncsa.daffodil.exceptions.Assert
* Maybe(null) = Nope
* One(null) == Nope // can't construct a Maybe type containing null.
*/
final class Maybe[+T <: AnyRef] /* @inline */ private (val v: Any) extends AnyVal {
final class Maybe[+T <: AnyRef](val v: Any) extends AnyVal {
import Maybe._
@inline final def get: T = if (isDefined) value else noneGet
@inline private final def value: T = v.asInstanceOf[T]
@inline final def value: T = v.asInstanceOf[T]
final def noneGet = throw new NoSuchElementException("Nope.get")

@inline final def isEmpty: Boolean = NopeValue eq v.asInstanceOf[AnyRef]
@@ -118,7 +118,7 @@ object Maybe {
//
// def unapply[T](value: Maybe[T]) = if (value.isDefined) scala.Some(value.get) else scala.None
}
private object NopeValue extends Serializable {
object NopeValue extends Serializable {
override def toString = "Nope"
}

@@ -49,7 +49,7 @@ class MyException(msg: String)
with OOLAGDiagnosticMixin

abstract class MyBase(parentArg: MyBase)
extends OOLAGHost(parentArg) {
extends OOLAGHost(parentArg) {

def a1 = a1_.value
def a1_ = LV('a1) {
@@ -73,7 +73,7 @@ abstract class MyBase(parentArg: MyBase)
}

class MySubHost(name: String, parent: MyBase)
extends MyBase(parent) {
extends MyBase(parent) {
requiredEvaluations(a1)
}

@@ -216,13 +216,13 @@ class TestOOLAG {
val h = new MyHost
var e: Exception = null
OOLAG.keepGoing() {
e = intercept[Exception] {
e = intercept[CircularDefinition] {
h.circ1
fail()
}
}
val msg = e.getMessage()
// println(e)
println(e)
assertTrue(msg.toLowerCase().contains("circ1"))
}

@@ -8,9 +8,10 @@ object LoggerMacros {
import c.universe._
q"""
{
val l = $lvl
if (getLoggingLevel() >= l)
doLogging(l, $msg, Seq(..$args))
val level = $lvl
val l = level.lvl
if (getLoggingLevel().lvl >= l)
doLogging(level, $msg, Seq(..$args))
}
"""
}
@@ -57,7 +57,7 @@ class StringDelimitedUnparser(erd: ElementRuntimeData,
try {
val valueString = theString(state)

val escapedValue =
val escapedValue: String =
if (schemeOpt.isDefined) {
state.withUnparserDataInputStream { dis =>
val inscopeDelimiters =
@@ -86,8 +86,8 @@ class StringDelimitedUnparser(erd: ElementRuntimeData,
val (result, _) = {
if (scheme.isInstanceOf[EscapeSchemeCharUnparserHelper]) {
val theScheme = scheme.asInstanceOf[EscapeSchemeCharUnparserHelper]
val thingsToEscape = inscopeDelimiters ++ scheme.lookingFor
val hasEscCharAsDelimiter = inscopeDelimiters.exists(d => d.lookingFor.length == 1 && d.lookingFor(0) =:= theScheme.ec.get)
val thingsToEscape = (inscopeDelimiters ++ scheme.lookingFor).toArray

textUnparser.escapeCharacter(dis, fieldDFA, thingsToEscape, hasEscCharAsDelimiter, theScheme.ec.get, theScheme.eec, state)
} else {
@@ -8,16 +8,18 @@ import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.util.Maybe._

object EmptyDelimiterStackNode {
val node = new DelimiterStackNode(Seq.empty, Seq.empty, Seq.empty, Nope, Nope, Nope)

private val mt = null // Array.empty[DFADelimiter] // mutable, so use null to avoid subtle bugs.
val node = new DelimiterStackNode(mt, mt, mt, Nope, Nope, Nope)
def apply() = node
}

object DelimiterStackNode {

def apply(
initiators: Seq[DFADelimiter],
separators: Seq[DFADelimiter],
terminators: Seq[DFADelimiter],
initiators: Array[DFADelimiter],
separators: Array[DFADelimiter],
terminators: Array[DFADelimiter],
initiatorLoc: Maybe[(String, String)],
separatorLoc: Maybe[(String, String)],
terminatorLoc: Maybe[(String, String)]): DelimiterStackNode = {
@@ -28,9 +30,9 @@ object DelimiterStackNode {
}

class DelimiterStackNode(
initiators: Seq[DFADelimiter],
separators: Seq[DFADelimiter],
terminators: Seq[DFADelimiter],
initiators: Array[DFADelimiter],
separators: Array[DFADelimiter],
terminators: Array[DFADelimiter],
initiatorLoc: Maybe[(String, String)],
separatorLoc: Maybe[(String, String)],
terminatorLoc: Maybe[(String, String)]) {
@@ -117,4 +119,4 @@ class DelimiterStackNode(
private lazy val separatorHasES = separators.find(i => i.lookingFor == "%ES;").isDefined
private lazy val terminatorHasES = terminators.find(i => i.lookingFor == "%ES;").isDefined

}
}
@@ -39,6 +39,9 @@ import edu.illinois.ncsa.daffodil.processors.unparsers.UState

trait Dynamic {

// TODO: Performance - we should consider avoiding using an Either object here since they are
// allocated every time this is called.

type CachedDynamic[A] = Either[CompiledExpression, A]

// Returns an Either, with Right being the value of the constant, and the
@@ -55,7 +58,12 @@ trait Dynamic {
}
}

def cacheConstantExpression[A](oe: Maybe[CompiledExpression])(conv: (Any) => A): Maybe[CachedDynamic[A]] = {
// Note: These method names used to be just overloads without the "Maybe" suffix.
// We don't really need them to be overloads, and some permutation of the Maybe[T] class
// with lots of inlining resulted in errors here because a Maybe[T] is an AnyVal aka
// value class. At compile time Maybe[Foo] and just Foo aren't distinguishable to resolve
// the overloading. So keep it simple, and just don't overload the names.
def cacheConstantExpressionMaybe[A](oe: Maybe[CompiledExpression])(conv: (Any) => A): Maybe[CachedDynamic[A]] = {
//oe.map { e => cacheConstantExpression[A](e)(conv) }
if (oe.isDefined) One(cacheConstantExpression[A](oe.get)(conv))
else Nope
@@ -89,7 +97,7 @@ trait Dynamic {
}
}

def evalWithConversion[A <: AnyRef](s: ParseOrUnparseState, oe: Maybe[CachedDynamic[A]])(conv: (ParseOrUnparseState, Any) => A): Maybe[A] = {
def evalWithConversionMaybe[A <: AnyRef](s: ParseOrUnparseState, oe: Maybe[CachedDynamic[A]])(conv: (ParseOrUnparseState, Any) => A): Maybe[A] = {
if (oe.isDefined) {
val a = evalWithConversion[A](s, oe.get)(conv)
One(a)
@@ -116,7 +124,7 @@ trait Dynamic {
}
}

def getStatic[A <: AnyRef](oe: Maybe[CachedDynamic[A]]): Maybe[A] = {
def getStaticMaybe[A <: AnyRef](oe: Maybe[CachedDynamic[A]]): Maybe[A] = {
if (oe.isDefined) getStatic(oe.get)
else Nope
}
@@ -35,7 +35,7 @@ case class EscapeSchemeBlockParserHelper(private val escEscChar: Maybe[String],
}

sealed abstract class EscapeSchemeUnparserHelper {
def lookingFor: Seq[DFADelimiter]
def lookingFor: Array[DFADelimiter]
}
case class EscapeSchemeCharUnparserHelper(private val escChar: Maybe[String], private val escEscChar: Maybe[String], private val extraEscChar: Maybe[Seq[String]])
extends EscapeSchemeUnparserHelper {
@@ -50,10 +50,10 @@ case class EscapeSchemeCharUnparserHelper(private val escChar: Maybe[String], pr
//
val escCharDFA: DFADelimiter = CreateDelimiterDFA(escChar.get)
val escEscCharDFA: Maybe[DFADelimiter] = if (escEscChar.isDefined) One(CreateDelimiterDFA(escEscChar.get)) else Nope
val extraEscCharsDFAs: Seq[DFADelimiter] = CreateDelimiterDFA(extraEscChar.getOrElse(Seq.empty))
val extraEscCharsDFAs: Array[DFADelimiter] = CreateDelimiterDFA(extraEscChar.getOrElse(Seq.empty))

override val lookingFor = {
val res: Seq[DFADelimiter] =
val res: Array[DFADelimiter] =
if (escEscCharDFA.isDefined) escCharDFA +: escCharDFA +: escEscCharDFA.get +: escEscCharDFA.get +: extraEscCharsDFAs
else escCharDFA +: escCharDFA +: extraEscCharsDFAs
res
@@ -79,7 +79,7 @@ case class EscapeSchemeBlockUnparserHelper(private val escEscChar: Maybe[String]
val blockEndDFA: DFADelimiter = CreateDelimiterDFA(blockEnd)
val blockStartDFA: DFADelimiter = CreateDelimiterDFA(blockStart)
val fieldEscDFA = CreateFieldDFA(blockEndDFA, eec)
val extraEscCharsDFAs: Seq[DFADelimiter] = CreateDelimiterDFA(extraEscChar.getOrElse(Seq.empty))
val extraEscCharsDFAs: Array[DFADelimiter] = CreateDelimiterDFA(extraEscChar.getOrElse(Seq.empty))

override val lookingFor = blockStartDFA +: extraEscCharsDFAs

@@ -23,7 +23,7 @@ trait EvaluatesStaticDynamicTextCommon { self: Processor =>
}
}

def combineStaticAndDynamic(static: Seq[DFADelimiter], dynamic: Seq[DFADelimiter]) = {
def combineStaticAndDynamic(static: Array[DFADelimiter], dynamic: Array[DFADelimiter]) = {
val res = static ++ dynamic
res
}
@@ -34,7 +34,7 @@ trait EvaluatesStaticDynamicTextParser

def getStaticAndDynamicText(ceOpt: Option[CompiledExpression],
context: RuntimeData,
errorOnWSPStar: Boolean = false): (Seq[DFADelimiter], Option[CompiledExpression]) = {
errorOnWSPStar: Boolean = false): (Array[DFADelimiter], Option[CompiledExpression]) = {

val res = if (ceOpt.isDefined) {
val ce = ceOpt.get
@@ -49,7 +49,7 @@ trait EvaluatesStaticDynamicTextParser

def getStaticAndDynamicText(ce: CompiledExpression,
context: RuntimeData,
errorOnWSPStar: Boolean): (Seq[DFADelimiter], Option[CompiledExpression]) = {
errorOnWSPStar: Boolean): (Array[DFADelimiter], Option[CompiledExpression]) = {

val (staticRawOpt, dynamicRawOpt) = {
if (ce.isConstant) {
@@ -59,7 +59,9 @@ trait EvaluatesStaticDynamicTextParser
}
val staticCooked: Queue[String] = new Queue
if (staticRawOpt.isDefined) {
staticRawOpt.get.foreach(x => staticCooked.enqueue(EntityReplacer { _.replaceAll(x, Some(context)) }))
EntityReplacer { er =>
staticRawOpt.get.foreach(x => staticCooked.enqueue(er.replaceAll(x, Some(context))))
}
}
if (errorOnWSPStar) errorIfDelimsHaveWSPStar(staticCooked, context)
val staticDFAs = CreateDelimiterDFA(staticCooked)
@@ -70,7 +72,7 @@ trait EvaluatesStaticDynamicTextParser
def evaluateDynamicText(dynamic: Option[CompiledExpression],
start: PState,
context: RuntimeData,
errorOnWSPStar: Boolean): Seq[DFADelimiter] = {
errorOnWSPStar: Boolean): Array[DFADelimiter] = {

if (dynamic.isDefined) {
val res = dynamic.get.evaluate(start)
@@ -81,7 +83,7 @@ trait EvaluatesStaticDynamicTextParser
val dfas = CreateDelimiterDFA(finalValueCooked)

dfas
} else { Seq.empty }
} else { Array.empty }
}
}

@@ -145,4 +147,4 @@ trait EvaluatesStaticDynamicTextUnparser
Some(dfas)
} else { None }
}
}
}
@@ -167,14 +167,15 @@ case class MPState() {
Assert.invariant(dsNode != null)
dsNode.getTerminatingMarkup
}
}.toList.toSeq // use list here because toSeq creates a lazy stream from an iterator
}.toArray

def getAllDelimitersWithPos = delimiterStack.iterator.flatMap {
dsNode =>
{
Assert.invariant(dsNode != null)
dsNode.getDelimitersWithPos
}
}.toList.toSeq
}.toArray
}

/**

0 comments on commit 57b2fce

Please sign in to comment.