Skip to content
Permalink
Browse files
First tests of outputValueCalc with real forward referencng working.
No regressions.

LayeredDataOutputStream, BufferedDataOutputStream, withBitLengthLimit

macro for withBitLengthLimit

ParseOrUnparseState.dState(expr) so every expression can have a DState
for unparsing.

SuspendableExpression - first cut

Revised BufferedDataOutputStream.

Eliminated LayeredDataOutputStream. Simplified "collapsing".
Renamed to reflect direct or buffered state.
Added unit tests.
Removed queue. Any BufferedDataOutputStream can give rise to only one
other new BufferedDataOutputStream.

Dynamic.scala - added a diatribe on why this complexity needs to
refactor/evolve into something cleaner and more minimal. This is only
used by text numbers, not universally. I'm not even sure these "caches"
are ever hit twice. It's less a cache than a runtime call to a converter
that can be converted at compilation time in the static case. (If so
rename at minimum)

Refactor and Simplify DataOutputStream.

For parallel structure, refactored DataInputStream also.

First OVC unit tests with real forward referencing working.
  • Loading branch information
mbeckerle committed Jan 25, 2016
1 parent cbae5d5 commit 8e6f00b04c610d2d8276c2d7a1d333334c6ccfa6
Showing 70 changed files with 2,078 additions and 951 deletions.
@@ -141,6 +141,9 @@ class ProcessorFactory(val sset: SchemaSet)
ExecutionMode.usingCompilerMode {
Assert.usage(canProceed)
if (xpath != "/") rootElem.notYetImplemented("""Path must be "/". Other path support is not yet implemented.""")
val rootERD = rootElem.elementRuntimeData
rootElem.schemaDefinitionUnless(rootERD.outputValueCalcExpr.isEmpty,
"The root element cannot have the dfdl:outputValueCalc property.")
val validationMode = ValidationMode.Off
val variables: VariableMap = rootElem.schemaDocument.schemaSet.variableMap
val p = if (rootElem.canProceed) parser else null
@@ -149,7 +152,7 @@ class ProcessorFactory(val sset: SchemaSet)
p,
u,
this.diagnostics,
rootElem.elementRuntimeData,
rootERD,
variables,
validationMode)
CheckJavaVersion.checkJavaVersion(ssrd)
@@ -634,14 +634,15 @@ trait ElementBaseGrammarMixin
ValueCalc("inputValueCalc", self, inputValueCalcOption), dfdlScopeEnd)
}

private lazy val ovcValueCalcObject = ValueCalc("outputValueCalc", self, outputValueCalcOption)
private lazy val ovcValueCalcObject =
ValueCalc("outputValueCalc", self, outputValueCalcOption, elementLeftFraming ~ scalarNonDefaultContent)

protected final lazy val ovcCompiledExpression = ovcValueCalcObject.expr

private lazy val outputValueCalcElement = prod("outputValueCalcElement",
isSimpleType && outputValueCalcOption.isInstanceOf[Found], forWhat = ForUnparser) {
new ElementCombinator(this,
dfdlScopeBegin ~ ovcValueCalcObject ~ elementLeftFraming ~ scalarNonDefaultContent,
dfdlScopeBegin ~ ovcValueCalcObject,
elementRightFraming ~ dfdlScopeEnd)
}

@@ -44,6 +44,7 @@ import edu.illinois.ncsa.daffodil.util.Maybe
import edu.illinois.ncsa.daffodil.dpath.DFDLCheckConstraintsFunction
import edu.illinois.ncsa.daffodil.processors.unparsers.Unparser
import edu.illinois.ncsa.daffodil.processors.unparsers.StatementElementUnparser
import edu.illinois.ncsa.daffodil.processors.unparsers.StatementElementOutputValueCalcUnparser
import edu.illinois.ncsa.daffodil.processors.unparsers.StatementElementUnparserNoRep
import edu.illinois.ncsa.daffodil.grammar.HasNoUnparser

@@ -74,11 +75,19 @@ class ElementCombinator(context: ElementBase, eGram: Gram, eAfterGram: Gram)
eParser,
eAfterParser)

override lazy val unparser: Unparser =
if (context.isRepresented)
new StatementElementUnparser(context.erd, context.name, uSetVar, eUnparser, eAfterUnparser)
else
new StatementElementUnparserNoRep(context.erd, context.name, uSetVar, eUnparser, eAfterUnparser)
override lazy val unparser: Unparser = {
if (context.isRepresented) {
if (context.isOutputValueCalc) {
new StatementElementOutputValueCalcUnparser(context.erd, context.name, uSetVar, eUnparser, eAfterUnparser)
} else {
new StatementElementUnparser(context.erd, context.name, uSetVar, eUnparser, eAfterUnparser)
}
} else {
// When "not represented" (meaning inputValueCalc), then we don't
// unparse anything. And we don't expect any infoset events either.
new StatementElementUnparserNoRep(context.erd, context.name, uSetVar)
}
}
}

class ChoiceElementCombinator(context: ElementBase, eGram: Gram, eAfterGram: Gram)
@@ -146,7 +155,7 @@ abstract class ElementCombinatorBase(context: ElementBase, eGram: Gram, eGramAft

def parser: Parser

lazy val uSetVar = context.setVariableStatements.map(_.gram.unparser)
lazy val uSetVar = context.setVariableStatements.map(_.gram.unparser).toArray
lazy val eUnparser: Maybe[Unparser] =
if (eGram.isEmpty) Maybe.Nope
else Maybe(eGram.unparser)
@@ -33,6 +33,7 @@
package edu.illinois.ncsa.daffodil.processors

import edu.illinois.ncsa.daffodil.grammar.Terminal
import edu.illinois.ncsa.daffodil.grammar.Gram
import edu.illinois.ncsa.daffodil.dsom._
import edu.illinois.ncsa.daffodil.dpath._
import edu.illinois.ncsa.daffodil.xml.XMLUtils
@@ -58,6 +59,7 @@ import edu.illinois.ncsa.daffodil.processors.unparsers.ElementOutputValueCalcUnp
import edu.illinois.ncsa.daffodil.processors.unparsers.SetVariableUnparser
import edu.illinois.ncsa.daffodil.processors.unparsers.NewVariableInstanceEndUnparser
import edu.illinois.ncsa.daffodil.processors.unparsers.NewVariableInstanceStartUnparser
import edu.illinois.ncsa.daffodil.processors.unparsers.Unparser
import edu.illinois.ncsa.daffodil.compiler.ForParser

abstract class AssertBase(decl: AnnotatedSchemaComponent,
@@ -200,14 +202,21 @@ abstract class ValueCalcBase(e: ElementBase)
case class ValueCalc(
override val baseName: String,
e: ElementBase,
property: PropertyLookupResult)
property: PropertyLookupResult,
ovcRepUnparserGram: Gram = null)
extends ValueCalcBase(e) {

val exprProp = property.asInstanceOf[Found]

def parser: DaffodilParser = new IVCParser(expr, e.elementRuntimeData)
def parser: DaffodilParser = {
Assert.usage(ovcRepUnparserGram eq null)
new IVCParser(expr, e.elementRuntimeData)
}

override def unparser: DaffodilUnparser = new ElementOutputValueCalcUnparser(e.elementRuntimeData)
override def unparser: DaffodilUnparser = {
val ovcRepUnparser = ovcRepUnparserGram.unparser
new ElementOutputValueCalcUnparser(e.elementRuntimeData, ovcRepUnparser)
}

}

@@ -105,6 +105,15 @@ case class StringOfSpecifiedLength(e: ElementBase) extends Terminal(e, true) wit
justificationTrim,
e.elementRuntimeData)

// TODO: PERFORMANCE The unparser below currently uses the worst-case algorithm
// which is what is needed for utf-8 where we don't know the length in bytes/bits until we
// actually convert the utf-8 to bytes.
// However, it does this even if the string is ascii, i.e., known fixed width.
//
// Either here or in the grammar, we should be choosing an algorithm that
// if the encoding is known, if it is fixed width then we just calculate the fixed length in bytes/bits
// directly rather than calling a method that outputs the string to bytes in order to measure it.

override lazy val unparser: Unparser =
new StringOfSpecifiedLengthUnparser(unparsingPadChar,
justificationPad,
@@ -13,6 +13,7 @@ import edu.illinois.ncsa.daffodil.processors.DISimple
import edu.illinois.ncsa.daffodil.processors.DIComplex
import edu.illinois.ncsa.daffodil.xml._
import edu.illinois.ncsa.daffodil.util.IteratorFromCursor
import edu.illinois.ncsa.daffodil.util.TestUtils

/*
* These are all tests of default-value insertion.
@@ -29,6 +30,11 @@ import edu.illinois.ncsa.daffodil.util.IteratorFromCursor
*/
class TestInfosetDefaultingInUnparser {

val xsd = XMLUtils.XSD_NAMESPACE
val dfdl = XMLUtils.dfdlAppinfoSource // XMLUtils.DFDL_NAMESPACE
val xsi = XMLUtils.XSI_NAMESPACE
val example = XMLUtils.EXAMPLE_NAMESPACE

@Test def testDefaultable = {
val sch = SchemaUtils.dfdlTestSchema(
<dfdl:format ref="tns:daffodilTest1"/>,
@@ -74,138 +80,4 @@ class TestInfosetDefaultingInUnparser {
assertEquals("Hello", afterFoo_s.dataValue)
}

@Test def testOutputValueCalc = {
val sch = SchemaUtils.dfdlTestSchema(
<dfdl:format ref="tns:daffodilTest1"/>,
<xs:element name="bar" dfdl:lengthKind="implicit">
<xs:complexType>
<xs:sequence>
<xs:element name="foo" type="xs:string" dfdl:outputValueCalc='{ "abcde" }' dfdl:lengthKind="explicit" dfdl:length="5"/>
<xs:element name="afterFoo" dfdl:lengthKind="explicit" dfdl:length="5" type="xs:string"/>
</xs:sequence>
</xs:complexType>
</xs:element>)
val compiler = Compiler()
val pf = compiler.compileNode(sch)
if (pf.isError) {
val msgs = pf.getDiagnostics.map(_.getMessage).mkString("\n")
throw new Exception(msgs)
}
val u = pf.onPath("/").asInstanceOf[DataProcessor]
if (u.isError) {
val msgs = u.getDiagnostics.map(_.getMessage).mkString("\n")
throw new Exception(msgs)
}

val source = Source.fromString(<bar xmlns={ XMLUtils.EXAMPLE_NAMESPACE }><afterFoo>Hello</afterFoo></bar>.toString)

val xmlEventCursor = new XMLEventCursorFromInput(source)
val rootERD = u.ssrd.elementRuntimeData

val is = Adapter(InfosetCursor.fromXMLEventCursor(xmlEventCursor, rootERD))

val Start(bar_s: DIComplex) = is.next; assertNotNull(bar_s)
val Start(foo_s: DISimple) = is.next
val End(foo_e: DISimple) = is.next; assertNotNull(foo_e)
val Start(afterFoo_s: DISimple) = is.next
val End(afterFoo_e: DISimple) = is.next; assertNotNull(afterFoo_e)
val End(bar_e: DIComplex) = is.next; assertNotNull(bar_e)

assertTrue(foo_s.asInstanceOf[DISimple].hasValue) // has a value because expression is a constant
assertTrue(foo_s.runtimeData.outputValueCalcExpr.isDefined)
assertEquals("Hello", afterFoo_s.dataValue)
}

@Test def testOutputValueCalcAfterOptional = {
val sch = SchemaUtils.dfdlTestSchema(
<dfdl:format ref="tns:daffodilTest1"/>,
<xs:element name="bar" dfdl:lengthKind="implicit">
<xs:complexType>
<xs:sequence>
<xs:element name="beforeFoo" type="xs:string" dfdl:initiator="beforeFoo" dfdl:lengthKind="explicit" dfdl:length="5" minOccurs="0"/>
<xs:element name="foo" type="xs:string" dfdl:outputValueCalc='{ "abcde" }' dfdl:lengthKind="explicit" dfdl:length="5"/>
<xs:element name="afterFoo" dfdl:lengthKind="explicit" dfdl:length="5" type="xs:string"/>
</xs:sequence>
</xs:complexType>
</xs:element>)
val compiler = Compiler()
val pf = compiler.compileNode(sch)
if (pf.isError) {
val msgs = pf.getDiagnostics.map(_.getMessage).mkString("\n")
throw new Exception(msgs)
}
val u = pf.onPath("/").asInstanceOf[DataProcessor]
if (u.isError) {
val msgs = u.getDiagnostics.map(_.getMessage).mkString("\n")
throw new Exception(msgs)
}

val source = Source.fromString(<bar xmlns={ XMLUtils.EXAMPLE_NAMESPACE }><afterFoo>Hello</afterFoo></bar>.toString)

val xmlEventCursor = new XMLEventCursorFromInput(source)
val rootERD = u.ssrd.elementRuntimeData

val is = Adapter(InfosetCursor.fromXMLEventCursor(xmlEventCursor, rootERD))

val Start(bar_s: DIComplex) = is.next; assertNotNull(bar_s)
val Start(foo_s: DISimple) = is.next
val End(foo_e: DISimple) = is.next; assertNotNull(foo_e)
val Start(afterFoo_s: DISimple) = is.next
val End(afterFoo_e: DISimple) = is.next; assertNotNull(afterFoo_e)
val End(bar_e: DIComplex) = is.next; assertNotNull(bar_e)

assertTrue(foo_s.asInstanceOf[DISimple].hasValue) // has a value because expression is a constant
assertEquals("abcde", foo_s.dataValue)
assertTrue(foo_s.runtimeData.outputValueCalcExpr.isDefined)
assertEquals("Hello", afterFoo_s.dataValue)
}

@Test def testMultipleOutputValueCalcAfterOptional = {
val sch = SchemaUtils.dfdlTestSchema(
<dfdl:format ref="tns:daffodilTest1"/>,
<xs:element name="bar" dfdl:lengthKind="implicit">
<xs:complexType>
<xs:sequence>
<xs:element name="beforeFoo" type="xs:string" dfdl:initiator="beforeFoo" dfdl:lengthKind="explicit" dfdl:length="5" minOccurs="0"/>
<xs:element name="foo" type="xs:string" dfdl:outputValueCalc='{ "abcde" }' dfdl:lengthKind="explicit" dfdl:length="5"/>
<xs:element name="foo2" type="xs:string" default="fghij" dfdl:lengthKind="delimited" dfdl:terminator="!"/>
<xs:element name="afterFoo" dfdl:lengthKind="explicit" dfdl:length="5" type="xs:string"/>
</xs:sequence>
</xs:complexType>
</xs:element>)
val compiler = Compiler()
val pf = compiler.compileNode(sch)
if (pf.isError) {
val msgs = pf.getDiagnostics.map(_.getMessage).mkString("\n")
throw new Exception(msgs)
}
val u = pf.onPath("/").asInstanceOf[DataProcessor]
if (u.isError) {
val msgs = u.getDiagnostics.map(_.getMessage).mkString("\n")
throw new Exception(msgs)
}

val source = Source.fromString(<bar xmlns={ XMLUtils.EXAMPLE_NAMESPACE }><afterFoo>Hello</afterFoo></bar>.toString)

val xmlEventCursor = new XMLEventCursorFromInput(source)
val rootERD = u.ssrd.elementRuntimeData

val is = Adapter(InfosetCursor.fromXMLEventCursor(xmlEventCursor, rootERD))

val Start(bar_s: DIComplex) = is.next; assertNotNull(bar_s)
val Start(foo_s: DISimple) = is.next
val End(foo_e: DISimple) = is.next; assertNotNull(foo_e)
val Start(foo2_s: DISimple) = is.next
val End(foo2_e: DISimple) = is.next; assertNotNull(foo2_e)
val Start(afterFoo_s: DISimple) = is.next
val End(afterFoo_e: DISimple) = is.next; assertNotNull(afterFoo_e)
val End(bar_e: DIComplex) = is.next; assertNotNull(bar_e)

assertEquals("foo2", foo2_s.erd.namedQName.local)
assertEquals("fghij", foo2_s.dataValueAsString)
assertTrue(foo_s.asInstanceOf[DISimple].hasValue) // constant expression, so has value.
assertTrue(foo_s.runtimeData.outputValueCalcExpr.isDefined)
assertEquals("Hello", afterFoo_s.dataValue)
}

}
@@ -47,6 +47,7 @@ import edu.illinois.ncsa.daffodil.Implicits._; object INoWarn2 { ImplicitsSuppre
import edu.illinois.ncsa.daffodil.processors.VariableMap
import edu.illinois.ncsa.daffodil.processors.PState
import edu.illinois.ncsa.daffodil.util.TestUtils
import edu.illinois.ncsa.daffodil.io.ByteBufferDataInputStream

class TestDFDLExpressionEvaluation extends Parsers {

@@ -68,7 +69,8 @@ class TestDFDLExpressionEvaluation extends Parsers {
val doc = Infoset.newDocument(erd)
doc.setRootElement(infosetRootElem)

val pstate = PState.createInitialPState(doc, erd, null, dp)
val dis = ByteBufferDataInputStream(java.nio.ByteBuffer.allocate(0), 0L) // fake. Zero bits available.
val pstate = PState.createInitialPState(doc, erd, dis, dp)
val result = compiledExpr.evaluate(pstate)
body(result)
}

0 comments on commit 8e6f00b

Please sign in to comment.