Skip to content
Permalink
Browse files
Establish the asymetry between DataInputStream and DataOutputStream
The primary contribution of this commit is to establish the asymetry
between DataInputStream and DataOutputStream with respect to the
positions and limits in the bit stream.

For the DataInputStream, we have this concept of the current bitPos0b,
and optionally there may be abound called bitLimit0b. There are 1b
variants of these.

For parsing, these are always absolute values, that is they contain bit
position relative the ultimate start of the input stream where parsing
began.

For DataOutputStream, we have slighly different concepts.

There are absolute and relative variants. The absolute bitPosOb or
absBitPos0b is symmetric to the parser's bitPos0b. It's the position
relative to the ultimate start of the output stream.

However, we often do not know this value. So the UState and
DataOutputStream have a maybeAbsBitPos0b which can be MaybeULong.Nope if
the value isn't known.

In addition we have the relative or relBitPos0b. This is relative to the
start of whatever buffer we are doing unparsing into.

When unparsing, we often have to unparse into a buffer where the
ultimate actual absolute position isn't yet known, but we have to do the
unparsing anyway, for example so that we can measure exactly how long
something is.

Conversely, sometimes we simply must have the absolute output bit
position, for example, when computing the number of bits to insert to
achieve the required alignment.

Hence we have relBitPos0b - always known and is a value >= 0, and we
have maybeAbsBitPos0b which is a MaybeULong. If known it is >=0.

Corresponding to bit position we have bit limit, which is measured in
the same 0b or 1b units, but is *always* a maybe type, because even in
the case where we know the absolute position, we still may or may not
have any limit in place. Hence the UState and DataOutputStream have a

maybeRelBitLimit0b
and
maybeAbsBitLimit0b.

One invariant is this: when the absolute bit pos is known, then it is
the same as the relative bit pos. Similarly when the absolute bit limit
is known, then the relative bit limit is known and is equal.

All deprecation warnings fixed.

No warnings now. Turned back on -Xfatal-warnings in scala compile
options.

There is still CachedDynamic in here, which is now using Evaluatable, so
we have the caches on top of the caches still.

Added missing maybeAbsBitPos0b and 1b methods.
  • Loading branch information
mbeckerle committed May 3, 2016
1 parent 940b5f4 commit 7577561c48b69b4b646167870bd341b1d8754c47
Showing 38 changed files with 678 additions and 310 deletions.
@@ -9,7 +9,7 @@ scalaVersion in ThisBuild := "2.11.8"
// incOptions := incOptions.value.withNameHashing(true) // 2.11 experimental incremental compilation improvements (perhaps not working right?)

scalacOptions in ThisBuild ++= Seq("-unchecked", "-deprecation", "-Yinline-warnings", "-Xxml:-coalescing", "-language:experimental.macros", "-Ybackend:GenBCode", "-Yopt-warnings",
"-Ywarn-inaccessible", "-Ywarn-unused-import", "-Ywarn-unused", "-Ywarn-infer-any", "-Ywarn-nullary-override", "-Ydead-code", "-Yopt:inline-global", "-Yinline" ) // "-Xfatal-warnings",
"-Ywarn-inaccessible", "-Ywarn-unused-import", "-Ywarn-unused", "-Ywarn-infer-any", "-Ywarn-nullary-override", "-Ydead-code", "-Yopt:inline-global", "-Yinline" , "-Xfatal-warnings")

// parallelExecution in ThisBuild := false

@@ -599,15 +599,15 @@ abstract class ElementBase(xmlArg: Node, parent: SchemaComponent, position: Int)
* be fixed or variable width.
*/
final lazy val isFixedLength = {
(lengthKind =:= LengthKind.Explicit && length.isConstant) ||
(lengthKind =:= LengthKind.Explicit && lengthEv.isConstant) ||
isImplicitLengthString
}

final def isImplicitLengthString = isSimpleType && primType =:= PrimType.String && lengthKind =:= LengthKind.Implicit

final lazy val fixedLengthValue: Long = {
Assert.usage(isFixedLength)
if (lengthKind =:= LengthKind.Explicit) length.constant
if (lengthKind =:= LengthKind.Explicit) lengthEv.optConstant.get
else {
Assert.invariant(lengthKind =:= LengthKind.Implicit)
// it's a string with implicit length. get from facets
@@ -649,7 +649,7 @@ abstract class ElementBase(xmlArg: Node, parent: SchemaComponent, position: Int)

// TODO: Is this used at all?
final lazy val fixedLength = {
if (isFixedLength) length.constant.longValue() else -1L // shouldn't even be asking for this if not isFixedLength
if (isFixedLength) lengthEv.optConstant.get.longValue() else -1L // shouldn't even be asking for this if not isFixedLength
}

/**
@@ -49,12 +49,6 @@ trait TermRuntimeValuedPropertiesMixin
extends DFDLBaseTypeMixin
with RawCommonRuntimeValuedPropertiesMixin { decl: Term =>

@deprecated("2016-03-10", "Use encodingEv instead.")
final lazy val encoding = {
encodingEv.compile()
encodingExpr
}

private lazy val encodingExpr = LV('encoding) {
val qn = this.qNameForProperty("encoding")
this match {
@@ -164,18 +158,16 @@ trait ElementRuntimeValuedPropertiesMixin
new ByteOrderEv(byteOrderExpr, elementRuntimeData)
}.value

@deprecated("2016-03-10", "Use lengthEv instead.")
final lazy val length = {
lengthEv.compile()
lengthExpr
}

private lazy val lengthExpr = {
val qn = this.qNameForProperty("length")
ExpressionCompilers.JLong.compile(qn, NodeInfo.Long, lengthRaw)
}

lazy val lengthEv = new LengthEv(lengthExpr, erd)
lazy val lengthEv = {
val ev = new LengthEv(lengthExpr, erd)
ev.compile()
ev
}

//
// The occursCount expression is written on the array element, but that expression
@@ -191,19 +183,17 @@ trait ElementRuntimeValuedPropertiesMixin
// implementation, the ".." does get literally evaluated.
//

@deprecated("2016-03-10", "Use occursCountEv instead.")
final lazy val occursCount = {
occursCountEv.compile()
occursCountExpr
}

private lazy val occursCountExpr = LV('occursCount) {
val qn = this.qNameForProperty("occursCount")
val isEvaluatedAbove = true
ExpressionCompilers.JLong.compile(qn, NodeInfo.Long, occursCountRaw, isEvaluatedAbove)
}.value

lazy val occursCountEv = new OccursCountEv(occursCountExpr, erd)
lazy val occursCountEv = {
val ev = new OccursCountEv(occursCountExpr, erd)
ev.compile()
ev
}
}

trait SequenceRuntimeValuedPropertiesMixin
@@ -236,68 +226,74 @@ trait SimpleTypeRuntimeValuedPropertiesMixin
extends DFDLSimpleTypeMixin
with RawSimpleTypeRuntimeValuedPropertiesMixin { decl: ElementBase =>

@deprecated("2016-03-10", "Use textStandardDecimalSeparatorEv instead.")
final lazy val textStandardDecimalSeparator = textStandardDecimalSeparatorExpr

private lazy val textStandardDecimalSeparatorExpr = LV('textStandardDecimalSeparator) {
val qn = this.qNameForProperty("textStandardDecimalSeparator")
val c = ExpressionCompilers.String.compile(qn, NodeInfo.String, textStandardDecimalSeparatorRaw)
c
}.value

final lazy val textStandardDecimalSeparatorEv = new TextStandardDecimalSeparatorEv(textStandardDecimalSeparatorExpr, erd)

@deprecated("2016-03-10", "Use textStandardGroupingSeparatorEv instead.")
final lazy val textStandardGroupingSeparator = textStandardGroupingSeparatorExpr
final lazy val textStandardDecimalSeparatorEv = {
val ev = new TextStandardDecimalSeparatorEv(textStandardDecimalSeparatorExpr, erd)
ev.compile()
ev
}

private lazy val textStandardGroupingSeparatorExpr = LV('textStandardGroupingSeparator) {
val qn = this.qNameForProperty("textStandardGroupingSeparator")
val c = ExpressionCompilers.String.compile(qn, NodeInfo.String, textStandardGroupingSeparatorRaw)
c
}.value

final lazy val textStandardGroupingSeparatorEv = new TextStandardGroupingSeparatorEv(textStandardGroupingSeparatorExpr, erd)

@deprecated("2016-03-10", "Use textStandardExponentRepEv instead.")
final lazy val textStandardExponentRep = textStandardExponentRepExpr
final lazy val textStandardGroupingSeparatorEv = {
val ev = new TextStandardGroupingSeparatorEv(textStandardGroupingSeparatorExpr, erd)
ev.compile()
ev
}

private lazy val textStandardExponentRepExpr = LV('textStandardExponentRep) {
val qn = this.qNameForProperty("textStandardExponentRep")
val c = ExpressionCompilers.String.compile(qn, NodeInfo.String, textStandardExponentRepRaw)
c
}.value

final lazy val textStandardExponentRepEv = new TextStandardExponentRepEv(textStandardExponentRepExpr, erd)

@deprecated("2016-03-10", "Use binaryFloatRepEv instead.")
final lazy val binaryFloatRep = binaryFloatRepExpr
final lazy val textStandardExponentRepEv = {
val ev = new TextStandardExponentRepEv(textStandardExponentRepExpr, erd)
ev.compile()
ev
}

private lazy val binaryFloatRepExpr = LV('binaryFloatRep) {
val qn = this.qNameForProperty("binaryFloatRep")
ExpressionCompilers.String.compile(qn, NodeInfo.NonEmptyString, binaryFloatRepRaw)
}.value

final lazy val binaryFloatRepEv = new BinaryFloatRepEv(binaryFloatRepExpr, erd)

@deprecated("2016-03-10", "Use textBooleanTrueRepEv instead.")
final lazy val textBooleanTrueRep = textBooleanTrueRepExpr
final lazy val binaryFloatRepEv = {
val ev = new BinaryFloatRepEv(binaryFloatRepExpr, erd)
ev.compile()
ev
}

private lazy val textBooleanTrueRepExpr = LV('textBooleanTrueRep) {
val qn = this.qNameForProperty("textBooleanTrueRep")
ExpressionCompilers.String.compile(qn, NodeInfo.NonEmptyString, textBooleanTrueRepRaw)
}.value

final lazy val textBooleanTrueRepEv = new TextBooleanTrueRepEv(textBooleanTrueRepExpr, erd)

@deprecated("2016-03-10", "Use textBooleanFalseRepEv instead.")
final lazy val textBooleanFalseRep = textBooleanFalseRepExpr
final lazy val textBooleanTrueRepEv = {
val ev = new TextBooleanTrueRepEv(textBooleanTrueRepExpr, erd)
ev.compile()
ev
}

private lazy val textBooleanFalseRepExpr = LV('textBooleanFalseRep) {
val qn = this.qNameForProperty("textBooleanFalseRep")
ExpressionCompilers.String.compile(qn, NodeInfo.NonEmptyString, textBooleanFalseRepRaw)
}.value

final lazy val textBooleanFalseRepEv = new TextBooleanFalseRepEv(textBooleanFalseRepExpr, erd)
final lazy val textBooleanFalseRepEv = {
val ev = new TextBooleanFalseRepEv(textBooleanFalseRepExpr, erd)
ev.compile()
ev
}

final lazy val calendarLanguage = LV('calendarLanguage) {
val qn = this.qNameForProperty("calendarLanguage")
@@ -25,9 +25,9 @@ trait TermEncodingMixin extends KnownEncodingMixin { self: Term =>
*/

final lazy val isKnownEncoding = {
val isKnown = encoding.isConstant
val isKnown = this.encodingEv.isConstant
if (isKnown) {
val encName = encoding.constant.toUpperCase()
val encName = encodingEv.optConstant.get.toUpperCase()
if (encName.startsWith("UTF-16")) {
utf16Width // demand this so checking is done
true
@@ -58,7 +58,7 @@ trait TermEncodingMixin extends KnownEncodingMixin { self: Term =>
}

lazy val encodingInfo =
new EncodingRuntimeData(termRuntimeData, encodingEv, decoderEv, encoderEv, schemaFileLocation, encoding, optionUTF16Width, defaultEncodingErrorPolicy,
new EncodingRuntimeData(termRuntimeData, encodingEv, decoderEv, encoderEv, schemaFileLocation, optionUTF16Width, defaultEncodingErrorPolicy,
summaryEncoding, isKnownEncoding, isScannable, knownEncodingAlignmentInBits)

/**
@@ -49,10 +49,10 @@ import java.lang.{ Long => JLong }
/////////////////////////////////////////////////////////////////

trait ElementBaseGrammarMixin
extends InitiatedTerminatedMixin
with AlignedMixin
with ByteOrderMixin
with HasStatementsGrammarMixin { self: ElementBase =>
extends InitiatedTerminatedMixin
with AlignedMixin
with ByteOrderMixin
with HasStatementsGrammarMixin { self: ElementBase =>

/**
* provided by LocalElementBase for array considerations, and GlobalElementDecl - scalar only
@@ -81,8 +81,8 @@ trait ElementBaseGrammarMixin

private lazy val binaryNumberKnownLengthInBits: Long = lengthKind match {
case LengthKind.Implicit => implicitBinaryLengthInBits
case LengthKind.Explicit if (length.isConstant) => {
val lengthFromProp: JLong = length.constant
case LengthKind.Explicit if (lengthEv.isConstant) => {
val lengthFromProp: JLong = lengthEv.optConstant.get
val nbits = lengthUnits match {
case LengthUnits.Bits => lengthFromProp.longValue()
case LengthUnits.Bytes => lengthFromProp.longValue() * 8
@@ -294,8 +294,8 @@ trait ElementBaseGrammarMixin
}

private lazy val staticBinaryFloatRep = {
subset(binaryFloatRep.isConstant, "Dynamic binaryFloatRep is not supported.")
BinaryFloatRep(binaryFloatRep.constant, this)
subset(binaryFloatRepEv.isConstant, "Dynamic binaryFloatRep is not supported.")
binaryFloatRepEv.optConstant.get
}

// private lazy val binary = {
@@ -35,21 +35,20 @@ package edu.illinois.ncsa.daffodil.grammar
import edu.illinois.ncsa.daffodil.processors._
import edu.illinois.ncsa.daffodil.dsom.Term
import edu.illinois.ncsa.daffodil.equality._
import edu.illinois.ncsa.daffodil.dsom.CompiledExpression

trait EncodingChangeMixin extends GrammarMixin { self: Term =>

private lazy val enclosingEncoding = enclosingTerm.map(_.encoding)
private lazy val priorSiblingEncoding = priorSibling.map(_.encoding)
private lazy val encodingBefore: CompiledExpression[String] = priorSiblingEncoding.getOrElse(enclosingEncoding.getOrElse(encoding))
private lazy val enclosingEncoding = enclosingTerm.map(_.encodingEv)
private lazy val priorSiblingEncoding = priorSibling.map(_.encodingEv)
private lazy val encodingBefore: Evaluatable[String] = priorSiblingEncoding.getOrElse(enclosingEncoding.getOrElse(encodingEv))

protected lazy val isKnownSameEncoding: Boolean = {
if (enclosingTerm.isEmpty) false // root always gets a encoding change.
else {
val thisEncoding = encoding
val thisEncoding = encodingEv
val otherEncoding = encodingBefore
if (thisEncoding.isConstant && otherEncoding.isConstant &&
thisEncoding.constant =#= otherEncoding.constant)
thisEncoding.optConstant.get =:= otherEncoding.optConstant.get)
true
else
false
@@ -45,8 +45,8 @@ import edu.illinois.ncsa.daffodil.processors.unparsers.DummyUnparser
import edu.illinois.ncsa.daffodil.util.Misc

class BinaryIntegerRuntimeLength(val e: ElementBase, signed: Boolean) extends Terminal(e, true) {
override lazy val parser = new BinaryIntegerRuntimeLengthParser(e.elementRuntimeData, signed, e.length, e.lengthUnits)
override lazy val parser = new BinaryIntegerRuntimeLengthParser(e.elementRuntimeData, signed, e.lengthEv, e.lengthUnits)

override lazy val unparser: Unparser = DummyUnparser(Misc.getNameFromClass(this))
}

@@ -57,8 +57,8 @@ class BinaryIntegerKnownLength(val e: ElementBase, signed: Boolean, lengthInBits
}

class BinaryDecimalRuntimeLength(val e: ElementBase) extends Terminal(e, true) {
override lazy val parser = new BinaryDecimalRuntimeLengthParser(e.elementRuntimeData, e.decimalSigned, e.binaryDecimalVirtualPoint, e.length, e.lengthUnits)
override lazy val parser = new BinaryDecimalRuntimeLengthParser(e.elementRuntimeData, e.decimalSigned, e.binaryDecimalVirtualPoint, e.lengthEv, e.lengthUnits)

override lazy val unparser: Unparser = DummyUnparser(Misc.getNameFromClass(this))
}

@@ -90,13 +90,13 @@ case class HexBinaryVariableLengthInBytes(e: ElementBase)
extends HexBinaryLengthInBytes(e) {

lazy val parserName = "HexBinaryVariableLengthInBytes"
lazy val lengthText = e.length.prettyExpr
lazy val lengthText = e.lengthEv.toString()

override lazy val parser: DaffodilParser = new HexBinaryVariableLengthInBytesParser(e.elementRuntimeData,
e.length)
e.lengthEv)

override lazy val unparser: DaffodilUnparser = new HexBinaryVariableLengthInBytesUnparser(e.elementRuntimeData,
e.length)
e.lengthEv)
}

case class StringOfSpecifiedLength(e: ElementBase) extends Terminal(e, true) with Padded {
@@ -143,7 +143,7 @@ abstract class StringDelimited(e: ElementBase)
if (!parsingPadChar.isDefined) None
else Some(new TextPaddingParser(parsingPadChar.get, e.elementRuntimeData))
}

// TODO: move out of parser and into the dsom
lazy val escapeSchemeParseEvOpt = if (es.isDefined) One(es.get.escapeSchemeParseEv) else Nope
lazy val escapeSchemeUnparseEvOpt = if (es.isDefined) One(es.get.escapeSchemeUnparseEv) else Nope
@@ -36,6 +36,7 @@ import edu.illinois.ncsa.daffodil.dsom._
import edu.illinois.ncsa.daffodil.grammar.Gram
import edu.illinois.ncsa.daffodil.grammar.Terminal
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.TextNumberRounding
import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.util.Maybe._
import java.math.BigInteger
import com.ibm.icu.text.DecimalFormat
@@ -99,31 +100,31 @@ abstract class ConvertTextNumberPrim[S](e: ElementBase)
(Nope, Nope)
}

val decSep =
val decSep: Maybe[Evaluatable[List[String]]] =
if (!h.isInt && (patternStripped.contains(".") ||
patternStripped.contains("E") ||
patternStripped.contains("@"))) {
One(e.textStandardDecimalSeparator)
One(e.textStandardDecimalSeparatorEv)
} else {
Nope
}

val groupSep =
if (patternStripped.contains(",")) {
One(e.textStandardGroupingSeparator)
One(e.textStandardGroupingSeparatorEv)
} else {
Nope
}

val isConstant = ((decSep.isEmpty || decSep.get.isConstant) &&
(groupSep.isEmpty || groupSep.get.isConstant) &&
e.textStandardExponentRep.isConstant)
e.textStandardExponentRepEv.isConstant)

val nff = if (isConstant) {
new NumberFormatFactoryStatic[S](e.runtimeData, h,
decSep,
groupSep,
e.textStandardExponentRep,
e.textStandardExponentRepEv,
infRep,
nanRep,
e.textNumberCheckPolicy,
@@ -135,7 +136,7 @@ abstract class ConvertTextNumberPrim[S](e: ElementBase)
new NumberFormatFactoryDynamic[S](e.runtimeData, h,
decSep,
groupSep,
e.textStandardExponentRep,
e.textStandardExponentRepEv,
infRep,
nanRep,
e.textNumberCheckPolicy,
@@ -142,7 +142,7 @@ object RepExactlyTotalOccursCount extends Rep2Arg(new RepExactlyTotalOccursCount
case class OccursCountExpression(e: ElementBase)
extends Terminal(e, true) {

override lazy val parser = new OccursCountExpressionParser(e.occursCount, e.elementRuntimeData)
override lazy val parser = new OccursCountExpressionParser(e.occursCountEv, e.elementRuntimeData)
override lazy val unparser = new NadaUnparser(e.runtimeData) // DFDL v1.0 Section 16.1.4 - The occursCount expression is not evaluated.

}

0 comments on commit 7577561

Please sign in to comment.