Skip to content
Permalink
Browse files
Improved data grammar to eliminate some asymmetries and complexity.
Fixes to io layer propagation of absolute positioning info.

Many cleanups to DataOutputStreams.

Lots of logging added to help debugging.
  • Loading branch information
mbeckerle committed Sep 2, 2016
1 parent f928bbd commit d60da53d37663626f5c986547bdc45797bf439a4
Showing 37 changed files with 1,007 additions and 758 deletions.

Large diffs are not rendered by default.

@@ -50,14 +50,17 @@ trait LocalElementGrammarMixin extends GrammarMixin { self: LocalElementBase =>

private lazy val notStopValue = prod("notStopValue", hasStopValue) { NotStopValue(this) }

private lazy val separatedEmpty = prod("separatedEmpty", emptyIsAnObservableConcept && isRecurring) { separatedForArrayPosition(empty) }
private lazy val separatedEmpty = prod("separatedEmpty",
emptyIsAnObservableConcept && isRecurring) {
separatedForArrayPosition(empty)
}

private lazy val separatedRecurringDefaultable = prod("separatedRecurringDefaultable", !isScalar) {
separatedForArrayPosition(enclosedElement)
}

private lazy val separatedRecurringNonDefault = prod("separatedRecurringNonDefault", !isScalar) {
separatedForArrayPosition(scalarNonDefault)
separatedForArrayPosition(enclosedElementNonDefault)
}

private lazy val nonSeparatedScalarDefaultable = prod("nonSeparatedScalarDefaultable", isScalar) { enclosedElement }
@@ -35,7 +35,6 @@ package edu.illinois.ncsa.daffodil.processors
import edu.illinois.ncsa.daffodil.dsom.ElementBase
import edu.illinois.ncsa.daffodil.equality.TypeEqual
import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.grammar.EmptyGram
import edu.illinois.ncsa.daffodil.grammar.Gram
import edu.illinois.ncsa.daffodil.grammar.HasNoUnparser
import edu.illinois.ncsa.daffodil.grammar.NamedGram
@@ -61,6 +60,7 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.NilKind
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.Representation
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.TestKind
import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.processors.unparsers.OVCRetryUnparser

/**
* This uber combinator exists because we (currently) do quite different things
@@ -76,29 +76,18 @@ import edu.illinois.ncsa.daffodil.util.Maybe
*/
class PhysicalElementUberCombinator(context: ElementBase,
eBeforeContent: Gram,
eValue: Gram // eAfterValue: Gram,
)
eValue: Gram,
eAfterValue: Gram)
extends NamedGram(context)
with Padded {

private lazy val uSetVars = context.setVariableStatements.map(_.gram.unparser).toArray

private lazy val beforeContent: Array[Unparser] =
if (eBeforeContent.isEmpty) Array()
else Array(eBeforeContent.unparser)

private lazy val value = Array(eValue.unparser)

private lazy val subComb = {
val eBeforeValue = EmptyGram
val eAfterValue = EmptyGram
val eAfterContent = EmptyGram
if (context.isParentUnorderedSequence) {
new ChoiceElementCombinator(context, eBeforeContent ~ eBeforeValue,
eValue, eAfterValue ~ eAfterContent)
new ChoiceElementCombinator(context, eBeforeContent,
eValue, eAfterValue)
} else {
new ElementCombinator(context, eBeforeContent ~ eBeforeValue,
eValue, eAfterValue ~ eAfterContent)
new ElementCombinator(context, eBeforeContent,
eValue, eAfterValue)
}
}

@@ -111,6 +100,14 @@ class PhysicalElementUberCombinator(context: ElementBase,
subComb.parser
}

private lazy val uSetVars = context.setVariableStatements.map(_.gram.unparser).toArray

private lazy val beforeContent: Array[Unparser] =
if (eBeforeContent.isEmpty) Array()
else Array(eBeforeContent.unparser)

private lazy val value = Array((eValue ~ eAfterValue).unparser)

override lazy val unparser: Unparser = {
if (context.isOutputValueCalc) {
new ElementOVCSpecifiedLengthUnparser(context.erd,
@@ -255,6 +252,14 @@ case class RightFill(ctxt: ElementBase)
ctxt.maybeUnparseTargetLengthInBitsEv.get, ctxt.fillByteEv, unparsingPadChar)
}

case class OVCRetry(ctxt: ElementBase, v: Gram)
extends Terminal(ctxt, true) {
override def parser = v.parser

override def unparser = new OVCRetryUnparser(ctxt.erd,
ctxt.maybeUnparseTargetLengthInBitsEv, v.unparser)
}

case class CaptureContentLengthStart(ctxt: ElementBase)
extends Terminal(ctxt, true) {
override def parser = new NadaParser(ctxt.erd)
@@ -39,33 +39,29 @@ import edu.illinois.ncsa.daffodil.exceptions.Assert
import edu.illinois.ncsa.daffodil.dsom.Found
import edu.illinois.ncsa.daffodil.dsom.NotFound

class DefaultablePhysicalOrComputed(ctxt: ElementBase,
scalarDefaultablePhysicalArg: => Gram,
class PhysicalOrComputed(ctxt: ElementBase,
scalarPhysicalArg: => Gram,
inputValueCalcElementArg: => Gram,
outputValueCalcElementArg: => Gram)
extends Terminal(ctxt, true) {
//Assert.invariant(!scalarDefaultablePhysical.isEmpty)
//Assert.invariant(!inputValueCalcElement.isEmpty)
//Assert.invariant(!outputValueCalcElement.isEmpty)
//Assert.invariant(!defaultableElement.isEmpty) // NYI? So this will be empty?

lazy val scalarDefaultablePhysical = scalarDefaultablePhysicalArg // once only
lazy val scalarPhysical = scalarPhysicalArg // once only
lazy val inputValueCalcElement = inputValueCalcElementArg // once only
lazy val outputValueCalcElement = outputValueCalcElementArg // once only

lazy val scalarDefaultablePhysicalParser = scalarDefaultablePhysical.parser
lazy val scalarPhysicalParser = scalarPhysical.parser
lazy val inputValueCalcElementParser = inputValueCalcElement.parser

lazy val inputValueCalcElementUnparser =
inputValueCalcElement.unparser
lazy val scalarDefaultablePhysicalUnparser = scalarDefaultablePhysical.unparser
lazy val scalarPhysicalUnparser = scalarPhysical.unparser
lazy val outputValueCalcElementUnparser = outputValueCalcElement.unparser

override lazy val parser = {
(ctxt.inputValueCalcOption, ctxt.outputValueCalcOption) match {
case (_: NotFound, _: Found) => scalarDefaultablePhysicalParser // outputValueCalc element is just a regular physical element for parser
case (_: NotFound, _: Found) => scalarPhysicalParser // outputValueCalc element is just a regular physical element for parser
case (_: Found, _: NotFound) => inputValueCalcElementParser
case (_: NotFound, _: NotFound) => scalarDefaultablePhysicalParser
case (_: NotFound, _: NotFound) => scalarPhysicalParser
case _ => Assert.impossibleCase()
}
}
@@ -80,7 +76,7 @@ class DefaultablePhysicalOrComputed(ctxt: ElementBase,
// element values may need to be in the infoset
case (_: Found, _: NotFound) =>
inputValueCalcElementUnparser
case _ => scalarDefaultablePhysicalUnparser
case _ => scalarPhysicalUnparser
}
}

@@ -53,10 +53,6 @@ import edu.illinois.ncsa.daffodil.processors.unparsers.SetVariableUnparser
import edu.illinois.ncsa.daffodil.processors.unparsers.NewVariableInstanceEndUnparser
import edu.illinois.ncsa.daffodil.processors.unparsers.NewVariableInstanceStartUnparser
import edu.illinois.ncsa.daffodil.compiler.ForParser
import edu.illinois.ncsa.daffodil.processors.unparsers.ElementOutputValueCalcRuntimeLengthUnparser
import edu.illinois.ncsa.daffodil.processors.unparsers.ElementOutputValueCalcStaticLengthUnparser
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.LengthUnits
import edu.illinois.ncsa.daffodil.util.MaybeULong

abstract class AssertBase(decl: AnnotatedSchemaComponent,
exprWithBraces: String,
@@ -213,59 +209,59 @@ case class InputValueCalc(e: ElementBase,
override lazy val unparser = Assert.usageError("Not to be called on InputValueCalc class.")
}

case class OutputValueCalcStaticLength(e: ElementBase,
property: PropertyLookupResult,
ovcRepUnparserGram: Gram,
knownLengthInBits: Long)
extends ValueCalcBase(e, property) {

override def baseName = "outputValueCalc"

override lazy val parser = Assert.usageError("Not to be called on OutputValueCalc class.")

override lazy val unparser = {
val ovcRepUnparser = ovcRepUnparserGram.unparser
val unp = new ElementOutputValueCalcStaticLengthUnparser(e.elementRuntimeData, ovcRepUnparser, MaybeULong(knownLengthInBits))
unp
}
}

case class OutputValueCalcRuntimeLength(e: ElementBase,
property: PropertyLookupResult,
ovcRepUnparserGram: Gram,
lengthEv: LengthEv,
lengthUnits: LengthUnits)
extends ValueCalcBase(e, property) {

override def baseName = "outputValueCalc"

override lazy val parser = Assert.usageError("Not to be called on OutputValueCalc class.")

override lazy val unparser = {
val ovcRepUnparser = ovcRepUnparserGram.unparser
val unp = new ElementOutputValueCalcRuntimeLengthUnparser(e.elementRuntimeData, ovcRepUnparser, lengthEv, lengthUnits)
unp
}
}

case class OutputValueCalcVariableLength(e: ElementBase,
property: PropertyLookupResult,
ovcRepUnparserGram: Gram)
extends ValueCalcBase(e, property) {

override def baseName = "outputValueCalc"

override lazy val parser = Assert.usageError("Not to be called on OutputValueCalc class.")

override lazy val unparser = {
val ovcRepUnparser = ovcRepUnparserGram.unparser
//
// same "static length" unparser, but the length is optional, so in this case we don't provide it.
//
val unp = new ElementOutputValueCalcStaticLengthUnparser(e.elementRuntimeData, ovcRepUnparser, MaybeULong.Nope)
unp
}
}
//case class OutputValueCalcStaticLength(e: ElementBase,
// property: PropertyLookupResult,
// ovcRepUnparserGram: Gram,
// knownLengthInBits: Long)
// extends ValueCalcBase(e, property) {
//
// override def baseName = "outputValueCalc"
//
// override lazy val parser = Assert.usageError("Not to be called on OutputValueCalc class.")
//
// override lazy val unparser = {
// val ovcRepUnparser = ovcRepUnparserGram.unparser
// val unp = new ElementOutputValueCalcStaticLengthUnparser(e.elementRuntimeData, ovcRepUnparser, MaybeULong(knownLengthInBits))
// unp
// }
//}
//
//case class OutputValueCalcRuntimeLength(e: ElementBase,
// property: PropertyLookupResult,
// ovcRepUnparserGram: Gram,
// lengthEv: LengthEv,
// lengthUnits: LengthUnits)
// extends ValueCalcBase(e, property) {
//
// override def baseName = "outputValueCalc"
//
// override lazy val parser = Assert.usageError("Not to be called on OutputValueCalc class.")
//
// override lazy val unparser = {
// val ovcRepUnparser = ovcRepUnparserGram.unparser
// val unp = new ElementOutputValueCalcRuntimeLengthUnparser(e.elementRuntimeData, ovcRepUnparser, lengthEv, lengthUnits)
// unp
// }
//}
//
//case class OutputValueCalcVariableLength(e: ElementBase,
// property: PropertyLookupResult,
// ovcRepUnparserGram: Gram)
// extends ValueCalcBase(e, property) {
//
// override def baseName = "outputValueCalc"
//
// override lazy val parser = Assert.usageError("Not to be called on OutputValueCalc class.")
//
// override lazy val unparser = {
// val ovcRepUnparser = ovcRepUnparserGram.unparser
// //
// // same "static length" unparser, but the length is optional, so in this case we don't provide it.
// //
// val unp = new ElementOutputValueCalcStaticLengthUnparser(e.elementRuntimeData, ovcRepUnparser, MaybeULong.Nope)
// unp
// }
//}

abstract class AssertPatternPrimBase(decl: AnnotatedSchemaComponent, stmt: DFDLAssertionBase)
extends Terminal(decl, true) {
@@ -37,9 +37,6 @@ import edu.illinois.ncsa.daffodil.grammar.Terminal
import edu.illinois.ncsa.daffodil.grammar.Gram
import edu.illinois.ncsa.daffodil.processors.{ Parser => DaffodilParser }
import edu.illinois.ncsa.daffodil.processors.unparsers.{ Unparser => DaffodilUnparser }

import edu.illinois.ncsa.daffodil.util.LogLevel

import edu.illinois.ncsa.daffodil.processors.dfa.TextPaddingParser
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.LengthKind
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.EscapeKind
@@ -164,16 +161,16 @@ abstract class StringDelimited(e: ElementBase)
}
}

/**
* Called at compile time in static case, at runtime for dynamic case.
*/
def errorIfDelimsHaveWSPStar(delims: List[String]): Unit = {
if (delims.filter(x => x == "%WSP*;").length > 0) {
// We cannot detect this error until expressions have been evaluated!
log(LogLevel.Debug, "%s - Failed due to WSP* detected as a delimiter for lengthKind=delimited.", eName)
context.schemaDefinitionError("WSP* cannot be used as a delimiter when lengthKind=delimited.")
}
}
// /**
// * Called at compile time in static case, at runtime for dynamic case.
// */
// def errorIfDelimsHaveWSPStar(delims: List[String]): Unit = {
// if (delims.filter(x => x == "%WSP*;").length > 0) {
// // We cannot detect this error until expressions have been evaluated!
// log(LogLevel.Debug, "%s - Failed due to WSP* detected as a delimiter for lengthKind=delimited.", eName)
// context.schemaDefinitionError("WSP* cannot be used as a delimiter when lengthKind=delimited.")
// }
// }

override lazy val parser: DaffodilParser = new StringDelimitedParser(
e.elementRuntimeData,
@@ -182,7 +179,12 @@ abstract class StringDelimited(e: ElementBase)
textDelimitedParser,
fieldDFAParseEv,
isDelimRequired)
override lazy val unparser: DaffodilUnparser = new StringDelimitedUnparser(e.elementRuntimeData, justificationPad, parsingPadChar, escapeSchemeUnparseEvOpt, isDelimRequired)
override lazy val unparser: DaffodilUnparser =
new StringDelimitedUnparser(e.elementRuntimeData,
justificationPad,
parsingPadChar,
escapeSchemeUnparseEvOpt,
isDelimRequired)

}

@@ -333,7 +333,8 @@ The message root is 'PCAP'.
then
(if (fn:exists(../../pcap:TransportLayer/pcap:UDP))
then dfdl:valueLength(../../pcap:TransportLayer/pcap:UDP/Data, 'bytes') + 20 + 8
else dfdl:valueLength(../../pcap:TransportLayer/pcap:TCP/Data, 'bytes') + dfdl:valueLength(../../pcap:TransportLayer/pcap:TCP/TCPHeader/Options, 'bytes') + 20 + 20
else dfdl:valueLength(../../pcap:TransportLayer/pcap:TCP/Data, 'bytes') +
dfdl:valueLength(../../pcap:TransportLayer/pcap:TCP/TCPHeader/Options, 'bytes') + 20 + 20
)
else if (fn:exists(../../pcap:ICMPv4/EchoRequest))
then dfdl:valueLength(../../pcap:ICMPv4/EchoRequest/Payload, 'bytes') + 20 + 8
@@ -39,14 +39,15 @@ import edu.illinois.ncsa.daffodil.exceptions.Assert
import java.nio.CharBuffer
import edu.illinois.ncsa.daffodil.util.Misc

private[io] class CharBufferDataOutputStreamState extends DataOutputStreamState
/**
* Used to implement unparsing when length is specified in lengthUnits 'characters' when
* the encoding is variable width (e.g., utf-8)
*/
final class CharBufferDataOutputStream
extends DataOutputStream with DataOutputStreamImplMixin {

override def id = 0

private[io] def isBuffering: Boolean = true

private def doNotUse = Assert.usageError("Not to be called on " + Misc.getNameFromClass(this))
@@ -60,8 +61,6 @@ final class CharBufferDataOutputStream
target = cb
}

protected override val st = new CharBufferDataOutputStreamState

private def notToBeUsed = Assert.usageError("not to be used")

override def putCharBuffer(cb: java.nio.CharBuffer): Long = {
@@ -37,6 +37,7 @@ import java.nio.charset.CharsetEncoder
import java.nio.ByteBuffer
import java.nio.CharBuffer
import edu.illinois.ncsa.daffodil.util.MaybeULong
import edu.illinois.ncsa.daffodil.util.Logging

/**
* There is an asymmetry between DataInputStream and DataOutputStream with respect to the
@@ -91,7 +92,10 @@ import edu.illinois.ncsa.daffodil.util.MaybeULong
* is known, then the relative bit limit is known and is equal.
*
*/
trait DataOutputStream extends DataStreamCommon {
trait DataOutputStream extends DataStreamCommon
with Logging {

def id: Int

def relBitPos0b: ULong

@@ -122,7 +126,7 @@ trait DataOutputStream extends DataStreamCommon {
/**
* sets, but also maintains the absolute bit limit, if that is defined.
*/
def setMaybeRelBitLimit0b(newMaybeRelBitLimit0b: MaybeULong): Boolean
def setMaybeRelBitLimit0b(newMaybeRelBitLimit0b: MaybeULong, reset: Boolean = false): Boolean

def resetMaybeRelBitLimit0b(savedBitLimit0b: MaybeULong): Unit

0 comments on commit d60da53

Please sign in to comment.