Skip to content
Permalink
Browse files
StringOfSpecifiedLengthUnparser renamed.
Code modified to reflect that this only truncates.

New name is StringMaybeTruncatedUnparser.

Fix center padding bug, remove unneeded & buggy padding code from prim.

Fix round trips that use utf-8, lengthUnits characters, padding and nil.

Remove debug variable

Remove ThinThrowable from UnparseError.

These are fatal errors, so we might as well get a decent backtrace
from them.

Removed problematic withBitLengthLimit on data output streams.

This concept is just broken for unparsing.

Enabled round trip on test.

Incorporated feedback on Review 662

DFDL-1600, DFDL-1325, DFDL-1599, DFDL-1579, DFDL-1603, DFDL-1557
  • Loading branch information
mbeckerle committed Sep 14, 2016
1 parent cea3c72 commit 49356e6fc4924e209cdd618e059e910efa698bbc
Showing 29 changed files with 609 additions and 386 deletions.
@@ -71,6 +71,10 @@ import edu.illinois.ncsa.daffodil.processors.MinLengthInBitsEv
import edu.illinois.ncsa.daffodil.processors.UnparseTargetLengthInBitsEv
import edu.illinois.ncsa.daffodil.processors.unparsers.NilStringLiteralForUnparserEv
import edu.illinois.ncsa.daffodil.processors.UnparseTargetLengthInCharactersEv
import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.processors.Evaluatable
import edu.illinois.ncsa.daffodil.util.MaybeJULong
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.NilKind

/*
* These are the DFDL properties which can have their values come
@@ -226,14 +230,14 @@ trait ElementRuntimeValuedPropertiesMixin
ExpressionCompilers.JLong.compile(qn, NodeInfo.Long, lengthRaw)
}

private lazy val explicitLengthEv = {
private lazy val explicitLengthEv: LengthEv = {
Assert.usage(lengthKind eq LengthKind.Explicit)
val ev = new ExplicitLengthEv(lengthExpr, erd)
ev.compile()
ev
}

private lazy val implicitLengthEv = {
private lazy val implicitLengthEv: LengthEv = {
Assert.usage(lengthKind eq LengthKind.Implicit)
import Representation._
import NodeInfo._
@@ -248,8 +252,6 @@ trait ElementRuntimeValuedPropertiesMixin
ev
}

// TODO: remove when no longer being used
// @deprecated("2016-08-11", "Use elementLengthInBitsEv instead. Converts to bits for you.")
final lazy val lengthEv = {
val ev =
lengthKind match {
@@ -262,6 +264,11 @@ trait ElementRuntimeValuedPropertiesMixin
ev
}

final lazy val maybeLengthEv: Maybe[LengthEv] = {
if (this.optionLengthRaw.isDefined) One(lengthEv)
else None
}

protected final lazy val optLengthConstant: Option[Long] = lengthEv.optConstant.map { _.longValue }

/**
@@ -310,22 +317,29 @@ trait ElementRuntimeValuedPropertiesMixin
ev
}

private lazy val (minLenUnits: LengthUnits, minLen: Long) = {
Assert.usage((lengthKind eq LengthKind.Implicit) || (lengthKind eq LengthKind.Explicit))
protected lazy val (minLenUnits: LengthUnits, minLen: Long) = {
import LengthKind._
import Representation._
import NodeInfo._
lazy val maxLengthLong = maxLength.longValueExact
lazy val minLengthLong = minLength.longValueExact
(lengthKind, impliedRepresentation, typeDef.kind) match {
case (Implicit, Binary, HexBinary) => (LengthUnits.Bytes, maxLengthLong) // fixed length
case (Implicit, Text, AnySimpleType) => (lengthUnits, textOutputMinLength) // fixed length
case (Implicit, Text, String) => (lengthUnits, maxLengthLong) // fixed length
case (Explicit, Text, String) => (lengthUnits, minLengthLong)
case (Explicit, Binary, HexBinary) => (LengthUnits.Bytes, minLengthLong)
case (Explicit, Text, AnySimpleType) => (lengthUnits, textOutputMinLength)
case _ => (LengthUnits.Bits, 0L) // anything else. This shuts off checking a min.
}
val res: (LengthUnits, Long) =
(lengthKind, impliedRepresentation, typeDef.kind) match {
case (Implicit, Binary, HexBinary) => (LengthUnits.Bytes, maxLengthLong) // fixed length
case (Implicit, Text, AnySimpleType) => (lengthUnits, textOutputMinLength) // fixed length
case (Implicit, Text, String) => (lengthUnits, maxLengthLong) // fixed length
case (Explicit, Text, String) => (lengthUnits, minLengthLong)
case (Explicit, Binary, HexBinary) => (LengthUnits.Bytes, minLengthLong)
case (Explicit, Text, AnySimpleType) => (lengthUnits, textOutputMinLength)
case (Prefixed, _, String) => (lengthUnits, minLengthLong)
case (Prefixed, Text, _) => (lengthUnits, textOutputMinLength)
case (Pattern, _, String) => (lengthUnits, minLengthLong)
case (Pattern, Text, _) => (lengthUnits, textOutputMinLength)
case (Delimited, _, String) => (lengthUnits, minLengthLong)
case (Delimited, Text, _) => (lengthUnits, textOutputMinLength)
case _ => (LengthUnits.Bits, 0L) // anything else. This shuts off checking a min.
}
res
}

final lazy val maybeUnparseTargetLengthInBitsEv = {
@@ -338,6 +352,26 @@ trait ElementRuntimeValuedPropertiesMixin
Nope
}

/**
* Used for padding, which might be for specified-length, or might be
* for delimited.
*/
final lazy val maybeUnparseMinOrTargetLengthInBitsEv: Maybe[Evaluatable[MaybeJULong]] = {
if ((this.optionLengthRaw.isDefined &&
(lengthKind _eq_ LengthKind.Explicit)) ||
((lengthKind _eq_ LengthKind.Implicit) && isSimpleType)) {
maybeUnparseTargetLengthInBitsEv
} else if (this.isDelimitedPrefixedPatternWithPadding) {
//
// if delimited but there is a min length, we just need the min
// length. There is no target length other than it.
//
val ev = minLengthInBitsEv
One(ev)
} else
Nope
}

final lazy val unparseTargetLengthInBitsEv = {
val ev = new UnparseTargetLengthInBitsEv(elementLengthInBitsEv, minLengthInBitsEv, erd)
ev.compile()
@@ -391,6 +425,13 @@ trait ElementRuntimeValuedPropertiesMixin
ev.compile()
ev
}

lazy val maybeLiteralNilEv = {
if (this.isNillable && (this.nilKind eq NilKind.LiteralValue))
One(nilStringLiteralForUnparserEv)
else
Nope
}
}

trait SequenceRuntimeValuedPropertiesMixin
@@ -62,7 +62,8 @@ trait ElementBaseGrammarMixin
private val context = this

private lazy val (leftPadding, rightPadding) = {
if (unparsingPadChar.isEmpty) (EmptyGram, EmptyGram)
if (unparsingPadChar.isEmpty)
(EmptyGram, EmptyGram)
else {
import TextJustificationType._
this.justificationPad match {
@@ -74,6 +75,28 @@ trait ElementBaseGrammarMixin
}
}

/**
* true if padding will be inserted for this delimited element when unparsing.
*/
protected lazy val isDelimitedPrefixedPatternWithPadding = {
import LengthKind._
val isRightLengthKind =
lengthKind match {
case Delimited => true // don't test for hasDelimiters because it might not be our delimiter, but a surrounding group's separator, or it's terminator, etc.
case Pattern => true
case Prefixed => true
case _ => false
}
(isRightLengthKind &&
(impliedRepresentation eq Representation.Text) &&
(justificationPad ne TextJustificationType.None) &&
minLen > 0)
}

lazy val shouldAddPadding =
maybeUnparseTargetLengthInBitsEv.isDefined ||
isDelimitedPrefixedPatternWithPadding

private lazy val rightFill = new RightFill(context)

// maybe can be private - if it is used still
@@ -206,18 +206,26 @@ case class ElementUnused(ctxt: ElementBase)
override def parser = new NadaParser(ctxt.erd)

override lazy val unparser: Unparser = new ElementUnusedUnparser(ctxt.erd,
ctxt.maybeUnparseTargetLengthInBitsEv.get, ctxt.fillByteEv)
ctxt.maybeUnparseTargetLengthInBitsEv.get,
ctxt.maybeLengthEv,
ctxt.maybeCharsetEv,
ctxt.maybeLiteralNilEv,
ctxt.fillByteEv)
}

case class OnlyPadding(ctxt: ElementBase)
extends Terminal(ctxt, ctxt.maybeUnparseTargetLengthInBitsEv.isDefined)
extends Terminal(ctxt, ctxt.shouldAddPadding)
with Padded {

override def parser = new NadaParser(ctxt.erd)

override lazy val unparser: Unparser =
new OnlyPaddingUnparser(ctxt.erd,
ctxt.maybeUnparseTargetLengthInBitsEv.get, unparsingPadChar)
ctxt.maybeUnparseMinOrTargetLengthInBitsEv.get,
ctxt.maybeLengthEv,
ctxt.maybeCharsetEv,
ctxt.maybeLiteralNilEv,
unparsingPadChar)
}

case class NilLiteralCharacter(ctxt: ElementBase)
@@ -232,27 +240,37 @@ case class NilLiteralCharacter(ctxt: ElementBase)
override lazy val unparser: Unparser =
new NilLiteralCharacterUnparser(ctxt.erd,
ctxt.maybeUnparseTargetLengthInBitsEv.get,
ctxt.maybeLengthEv,
ctxt.maybeCharsetEv,
nilLitCharacter)
}

case class RightCenteredPadding(ctxt: ElementBase)
extends Terminal(ctxt, ctxt.maybeUnparseTargetLengthInBitsEv.isDefined)
extends Terminal(ctxt, ctxt.shouldAddPadding)
with Padded {
override def parser = new NadaParser(ctxt.erd)

override lazy val unparser: Unparser =
new RightCenteredPaddingUnparser(ctxt.erd,
ctxt.maybeUnparseTargetLengthInBitsEv.get, unparsingPadChar)
ctxt.maybeUnparseMinOrTargetLengthInBitsEv.get,
ctxt.maybeLengthEv,
ctxt.maybeCharsetEv,
ctxt.maybeLiteralNilEv,
unparsingPadChar)
}

case class LeftCenteredPadding(ctxt: ElementBase)
extends Terminal(ctxt, ctxt.maybeUnparseTargetLengthInBitsEv.isDefined)
extends Terminal(ctxt, ctxt.shouldAddPadding)
with Padded {
override def parser = new NadaParser(ctxt.erd)

override lazy val unparser: Unparser =
new LeftCenteredPaddingUnparser(ctxt.erd,
ctxt.maybeUnparseTargetLengthInBitsEv.get, unparsingPadChar)
ctxt.maybeUnparseMinOrTargetLengthInBitsEv.get,
ctxt.maybeLengthEv,
ctxt.maybeCharsetEv,
ctxt.maybeLiteralNilEv,
unparsingPadChar)
}

case class RightFill(ctxt: ElementBase)
@@ -261,7 +279,12 @@ case class RightFill(ctxt: ElementBase)
override def parser = new NadaParser(ctxt.erd)

override lazy val unparser: Unparser = new RightFillUnparser(ctxt.erd,
ctxt.maybeUnparseTargetLengthInBitsEv.get, ctxt.fillByteEv, unparsingPadChar)
ctxt.maybeUnparseTargetLengthInBitsEv.get,
ctxt.maybeLengthEv,
ctxt.maybeCharsetEv,
ctxt.maybeLiteralNilEv,
ctxt.fillByteEv,
unparsingPadChar)
}

case class OVCRetry(ctxt: ElementBase, v: Gram)
@@ -300,7 +323,7 @@ case class CaptureValueLengthStart(ctxt: ElementBase)
//
// For all other elements, we can just use the Capture*ValueLength parsers.
if ((ctxt.isSimpleType && ctxt.impliedRepresentation == Representation.Text) ||
(ctxt.isComplexType && ctxt.lengthKind != LengthKind.Implicit)) new NadaParser(ctxt.erd)
(ctxt.isComplexType && ctxt.lengthKind != LengthKind.Implicit)) new NadaParser(ctxt.erd)
else new CaptureStartOfValueLengthParser(ctxt.erd)
}

@@ -320,7 +343,7 @@ case class CaptureValueLengthEnd(ctxt: ElementBase)
//
// For all other elements, we can just use the Capture*ValueLength parsers.
if ((ctxt.isSimpleType && ctxt.impliedRepresentation == Representation.Text) ||
(ctxt.isComplexType && ctxt.lengthKind != LengthKind.Implicit)) new NadaParser(ctxt.erd)
(ctxt.isComplexType && ctxt.lengthKind != LengthKind.Implicit)) new NadaParser(ctxt.erd)
else new CaptureEndOfValueLengthParser(ctxt.erd)
}

@@ -44,6 +44,7 @@ import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.TextTrimKind
import edu.illinois.ncsa.daffodil.util.MaybeChar
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.YesNo
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.Representation
import edu.illinois.ncsa.daffodil.schema.annotation.props.gen.LengthKind

trait PaddingInfoMixin {
def eBase: ElementBase
@@ -62,23 +63,32 @@ trait PaddingInfoMixin {
case TextTrimKind.PadChar if eBase.isSimpleType => padCharAndJustificationForType
}

lazy val (unparsingPadChar: MaybeChar, justificationPad) = eBase.textPadKind match {
case TextPadKind.None => (MaybeChar.Nope, TextJustificationType.None)
case TextPadKind.PadChar if eBase.isSimpleType &&
eBase.impliedRepresentation == Representation.Text => padCharAndJustificationForType
case _ => (MaybeChar.Nope, TextJustificationType.None)
lazy val (unparsingPadChar: MaybeChar, justificationPad) = {
val tpk = eBase.textPadKind
val res = tpk match {
case TextPadKind.None => (MaybeChar.Nope, TextJustificationType.None)
case TextPadKind.PadChar if eBase.isSimpleType &&
eBase.impliedRepresentation == Representation.Text => padCharAndJustificationForType
case _ => (MaybeChar.Nope, TextJustificationType.None)
}
res
}

lazy val stringTruncationType: TextTruncationType.Type =
if (eBase.primType != PrimType.String) TextTruncationType.None
else if (eBase.truncateSpecifiedLengthString eq YesNo.No) TextTruncationType.None
else eBase.textStringJustification match {
case TextStringJustification.Left => TextTruncationType.Left
case TextStringJustification.Right => TextTruncationType.Right
case TextStringJustification.Center => {
eBase.SDE("Properties dfdl:truncateSpecifiedLengthString 'yes' and dfdl:textStringJustification 'center' are incompatible.")
lazy val stringTruncationType: TextTruncationType.Type = {
val res =
if (eBase.primType != PrimType.String) TextTruncationType.None
else if (eBase.truncateSpecifiedLengthString eq YesNo.No) TextTruncationType.None
else if ((eBase.lengthKind eq LengthKind.Pattern) ||
(eBase.lengthKind eq LengthKind.Prefixed)) TextTruncationType.None
else eBase.textStringJustification match {
case TextStringJustification.Left => TextTruncationType.Left
case TextStringJustification.Right => TextTruncationType.Right
case TextStringJustification.Center => {
eBase.SDE("Properties dfdl:truncateSpecifiedLengthString 'yes' and dfdl:textStringJustification 'center' are incompatible.")
}
}
}
res
}

private lazy val padCharAndJustificationForType: (MaybeChar, TextJustificationType.Type) = {
val theJust = eBase.primType match {

0 comments on commit 49356e6

Please sign in to comment.