Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ object DeprecatedProperty {
DeprecatedProperty(XMLUtils.DFDL_NAMESPACE, "layerLength", "dfdlx:layerLength"),
DeprecatedProperty(XMLUtils.DFDL_NAMESPACE, "layerLengthUnits", "dfdlx:layerLengthUnits"),
DeprecatedProperty(XMLUtils.DFDL_NAMESPACE, "layerBoundaryMark", "dfdlx:layerBoundaryMark"),
DeprecatedProperty(XMLUtils.DFDLX_NAMESPACE, "emptyElementParsePolicy", "dfdl:emptyElementParsePolicy"),
DeprecatedProperty(XMLUtils.EXT_NS_APACHE, "parseUnparsePolicy", "dfdlx:parseUnparsePolicy"),
DeprecatedProperty(XMLUtils.EXT_NS_NCSA, "parseUnparsePolicy", "dfdlx:parseUnparsePolicy"))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -438,11 +438,11 @@ class ScalarOrderedSequenceChild(sq: SequenceTermBase, term: Term, groupIndex: I

/**
* Must deal with nils, emptyness and string/hexBinary exceptional behavior
* including the behavior for dfdlx:emptyElementParsePolicy 'treatAsMissing' which special cases
* including the behavior for dfdl:emptyElementParsePolicy 'treatAsAbsent' which special cases
* Required elements like scalars, iff they are emptyRep, emptyValueDelimiterPolicy,
* nilValueDelimiterPolicy, complex elements that nillable, or fully defaultable.
*
* So we have ((simpleStringHexBinary x (treatAsMissing, treatAsEmpty), simpleOther, complex) x (nillable, not) x
* So we have ((simpleStringHexBinary x (treatAsAbsent, treatAsEmpty), simpleOther, complex) x (nillable, not) x
* 4 behaviors. That's 32 combinations. Let's start with fewer cases and more runtime
* decisions, and specialize if we think it will help clarity or performance.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@
</dfdl:defineFormat>

<dfdl:defineFormat name="GeneralFormat">
<dfdl:format ref="GeneralFormatOriginal" />
<dfdl:format ref="GeneralFormatOriginal"
emptyElementParsePolicy="treatAsEmpty"/>
</dfdl:defineFormat>

<dfdl:defineFormat name="GeneralFormatPortable">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -430,9 +430,11 @@ trait TextStandardExponentRepMixin extends PropertyMixin {
*/
sealed trait EmptyElementParsePolicy extends EmptyElementParsePolicy.Value
object EmptyElementParsePolicy extends Enum[EmptyElementParsePolicy] {
case object TreatAsMissing extends EmptyElementParsePolicy
case object TreatAsMissing extends EmptyElementParsePolicy // deprecated
case object TreatAsEmpty extends EmptyElementParsePolicy
override lazy val values = Array(TreatAsMissing, TreatAsEmpty)
case object TreatAsAbsent extends EmptyElementParsePolicy

override lazy val values = Array(TreatAsMissing, TreatAsEmpty, TreatAsAbsent) // deprecated: TreatAsMissing

def apply(name: String, context: ThrowsSDE): EmptyElementParsePolicy = stringToEnum("emptyElementParsePolicy", name, context)
}
Expand All @@ -454,7 +456,7 @@ trait EmptyElementParsePolicyMixin extends PropertyMixin {

/**
* Property determines whether Daffodil implements empty elements in a manner consistent with
* IBM DFDL (as of 2019-05-02), which has policy treatAsMissing, or implements what is
* IBM DFDL (as of 2019-05-02), which has policy treatAsAbsent, or implements what is
* described in the DFDL spec., which is treatAsEmpty - if the syntax of
* empty (or nullness) is matched, create an empty (or null) item, even if optional, unless
* the element is entirely absent.
Expand All @@ -468,15 +470,10 @@ trait EmptyElementParsePolicyMixin extends PropertyMixin {
// prop is not required AND not defined so use tunable value
// but issue warning (which can be suppressed)
val defaultEmptyElementParsePolicy = this.tunable.defaultEmptyElementParsePolicy
// This property is an extension, so we don't want to require users to
// add this property just to silence this warning, especially since the
// property might change in the future. So silently use a default without
// outputting a warning. We may want to turn this warning on when the
// property makes its way into the official DFDL spec.
//SDW(
// WarnID.EmptyElementParsePolicyError,
// "Property 'dfdlx:emptyElementParsePolicy' is required but not defined, using tunable '%s' by default.",
// defaultEmptyElementParsePolicy)
SDW(
WarnID.EmptyElementParsePolicyError,
"Property 'dfdl:emptyElementParsePolicy' is required but not defined, using tunable '%s' by default.",
defaultEmptyElementParsePolicy)
defaultEmptyElementParsePolicy
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,13 @@
</xsd:restriction>
</xsd:simpleType>

<xsd:simpleType name="EmptyElementParsePolicyEnum">
<xsd:restriction base="xsd:string">
<xsd:enumeration value="treatAsEmpty" />
<xsd:enumeration value="treatAsAbsent" />
</xsd:restriction>
</xsd:simpleType>

<!-- ============================================================= -->
<!-- Simple types with DFDL Expression -->
<!-- ============================================================= -->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,9 @@
<xsd:attribute name="bitOrder" type="dfdl:BitOrderEnum"/>
<xsd:attribute ref="daf:parseUnparsePolicy"/> <!-- backwards compatibility -->
<xsd:attribute ref="dfdlx:parseUnparsePolicy"/>
<xsd:attribute ref="dfdlx:emptyElementParsePolicy"/>

<xsd:attribute name="emptyElementParsePolicy" type="dfdl:EmptyElementParsePolicyEnum"/>
<xsd:attribute ref="dfdlx:emptyElementParsePolicy"/> <!-- deprecated -->

<xsd:attribute name="encoding"
type="dfdl:EncodingEnum_Or_DFDLExpression" />
<xsd:attribute name="utf16Width" type="dfdl:UTF16WidthEnum" />
Expand Down Expand Up @@ -539,6 +540,7 @@
<xsd:enumeration value="bitOrder"/>
<xsd:enumeration value="encoding" />
<xsd:enumeration value="encodingErrorPolicy"/>
<xsd:enumeration value="emptyElementParsePolicy"/>
<xsd:enumeration value="utf16Width" />
<xsd:enumeration value="ignoreCase" />

Expand Down Expand Up @@ -641,6 +643,8 @@
type="dfdl:EncodingEnum_Or_DFDLExpression" />
<xsd:attribute form="qualified" name="encodingErrorPolicy"
type="dfdl:EncodingErrorPolicyEnum" />
<xsd:attribute form="qualified" name="emptyElementParsePolicy"
type="dfdl:EmptyElementParsePolicyEnum" />
<xsd:attribute form="qualified" name="utf16Width"
type="dfdl:UTF16WidthEnum" />
<xsd:attribute form="qualified" name="ignoreCase"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -598,7 +598,11 @@
</xs:element>

<xs:simpleType name="TunableEmptyElementParsePolicy">
<xs:restriction base="dfdlx:EmptyElementParsePolicyEnum" />
<xs:restriction base="xs:string">
<xs:enumeration value="treatAsEmpty" />
<xs:enumeration value="treatAsAbsent" />
<xs:enumeration value="treatAsMissing" /> <!-- deprecated -->
</xs:restriction>
</xs:simpleType>

<!--
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
<xs:restriction base="xs:string">
<xs:enumeration value="dfdlx:alignmentKind" />
<xs:enumeration value="dfdlx:choiceBranchKeyRanges" />
<xs:enumeration value="dfdlx:emptyElementParsePolicy"/>
<xs:enumeration value="dfdlx:emptyElementParsePolicy"/> <!-- deprecated -->
<xs:enumeration value="dfdlx:inputTypeCalc"/>
<xs:enumeration value="dfdlx:objectKind"/>
<xs:enumeration value="dfdlx:outputTypeCalc"/>
Expand Down Expand Up @@ -94,11 +94,11 @@
</xs:restriction>
</xs:simpleType>

<xs:attribute name="emptyElementParsePolicy" type="dfdlx:EmptyElementParsePolicyEnum"/>
<xs:attribute name="emptyElementParsePolicy" type="dfdlx:EmptyElementParsePolicyEnum"/> <!-- deprecated -->
<xs:simpleType name="EmptyElementParsePolicyEnum">
<xs:restriction base="xs:string">
<xs:enumeration value="treatAsEmpty" />
<xs:enumeration value="treatAsMissing" />
<xs:enumeration value="treatAsMissing" /> <!-- deprecated -->
</xs:restriction>
</xs:simpleType>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,11 @@ object ParseAttemptStatus {
*
* The EmptyRep for simpleTypes enables default values to be substituted at parse time.
*
* For simple types xs:string and xs:hexBinary, the property dfdlx:emptyElementParsePolicy controls
* For simple types xs:string and xs:hexBinary, the property dfdl:emptyElementParsePolicy controls
* whether the EmptyRep is allowed for strings and hexBinary. In required positions, when
* dfdlx:emptyElementParsePolicy is 'treatAsMissing', a required string/hexBinary that has EmptyRep
* dfdl:emptyElementParsePolicy is 'treatAsAbsent', a required string/hexBinary that has EmptyRep
* causes a Parse Error, and an optional EmptyRep causes nothing to be added to the infoset (the empty string
* or hexBinary value is suppressed). When dfdlx:emptyElementParsePolicy is 'treatAsEmpty', a required
* or hexBinary value is suppressed). When dfdl:emptyElementParsePolicy is 'treatAsEmpty', a required
* string/hexBinary with EmptyRep creates an empty string or zero-length byte array in the infoset.
* An optional EmptyRep behaves differently depending on whether the EmptyRep is truly zero-length, or
* dfdl:emptyValueDelimiterPolicy is such that EmptyRep is non-zero-length. When truly zero-length, no
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ trait ElementSequenceChildParseResultHelper
pstate.schemaDefinitionError("Default values not implemented.")
} else {
emptyElementParsePolicy match {
case EmptyElementParsePolicy.TreatAsMissing => {
case EmptyElementParsePolicy.TreatAsMissing | EmptyElementParsePolicy.TreatAsAbsent => { // deprecated: TreatAsMissing
parser.PE(pstate, "Empty element not allowed for required element.")
ParseAttemptStatus.MissingItem
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@

<tdml:defineConfig name="cfg_noEmptyElements">
<daf:tunables>
<daf:defaultEmptyElementParsePolicy>treatAsMissing</daf:defaultEmptyElementParsePolicy>
<daf:defaultEmptyElementParsePolicy>treatAsAbsent</daf:defaultEmptyElementParsePolicy>
<daf:suppressSchemaDefinitionWarnings>
unsupportedAttributeFormDefault
encodingErrorPolicyError
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@
</daf:suppressSchemaDefinitionWarnings>
</daf:tunables>
</tdml:defineConfig>

<tdml:defineConfig name="cfg_noEmptyElements">
<daf:tunables>
<daf:defaultEmptyElementParsePolicy>treatAsMissing</daf:defaultEmptyElementParsePolicy>
<daf:defaultEmptyElementParsePolicy>treatAsAbsent</daf:defaultEmptyElementParsePolicy>
<daf:suppressSchemaDefinitionWarnings>
unsupportedAttributeFormDefault
unsupportedAttributeFormDefault
encodingErrorPolicyError
</daf:suppressSchemaDefinitionWarnings>
</daf:tunables>
Expand Down Expand Up @@ -209,7 +209,7 @@
<record>
<required1>a</required1>
<!--
On Daffodil, because we are using dfdlx:emptyElementParsePolicy='treatAsMissing'
On Daffodil, because we are using dfdl:emptyElementParsePolicy='treatAsAbsent'
This test should not create any empty-string-valued elements.
That should be consistent with IBM DFDL behavior (as of 2019-05-04 version).
<positional2/>
Expand Down Expand Up @@ -247,7 +247,7 @@
<ptg1>
<record>
<required1>a</required1>
<!-- dfdlx:emptyElementParsePolicy='treatAsMissing'
<!-- dfdl:emptyElementParsePolicy='treatAsAbsent'
<positional2/>
<positional3/>
-->
Expand All @@ -257,7 +257,7 @@
</record>
<record>
<required1>a</required1>
<!-- dfdlx:emptyElementParsePolicy='treatAsMissing'
<!-- dfdl:emptyElementParsePolicy='treatAsAbsent'
<positional2/>
<positional3/>
-->
Expand All @@ -283,7 +283,7 @@
<!--
Because these are optional, and their dfdl:emptyValueDelimiterPolicy is 'none'
an empty representation does not create an element.
This is true for dfdlx:emptyElementParsePolicy="treatAsEmpty".
This is true for dfdl:emptyElementParsePolicy="treatAsEmpty".
If you want empty elements, there has to be a non-zero-length
syntax in the data stream
<positional2/>
Expand Down Expand Up @@ -640,7 +640,69 @@
</xs:element>

</tdml:defineSchema>


<!-- Same as s3 schema but emptyElementParsePolicy is now treatAsAbsent -->

<tdml:defineSchema name="s3_absent" elementFormDefault="unqualified">
<xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd" />

<dfdl:format ref="ex:GeneralFormatPortable"
lengthKind="delimited"
occursCountKind="implicit"
emptyElementParsePolicy="treatAsAbsent"/>

<!--
ptg3 explores spoiling the choice's potentially trailing by just having
a single branch that has required framing.
-->
<xs:element name="ptg3" dfdl:initiator="|">
<xs:complexType>
<xs:sequence dfdl:separator="">
<xs:element dfdl:occursCountKind="implicit" dfdl:terminator="|%WSP*;"
maxOccurs="unbounded" name="record">
<xs:complexType>
<xs:sequence dfdl:separator="," dfdl:separatorSuppressionPolicy="trailingEmpty">
<xs:element name="required1" type="xs:string" />
<xs:element name="positional2" type="xs:string" minOccurs="0" /> <!-- never trailing due to choice -->
<xs:element name="positional3" type="xs:string" minOccurs="0" /> <!-- never trailing due to choice -->
<xs:choice>
<!--
choice is not trailing, because none of the choice branches are.
(In general, it ought to be the last choice branch, otherwise
finding zero length will cut off consideration of any branches after
the first potentially trailing branch.)
-->
<xs:sequence>
<xs:element name="positional4_n" type="xs:decimal"/>
</xs:sequence>
<xs:sequence>
<xs:element name="positional4_s" type="xs:string" />
</xs:sequence>
<xs:sequence />
</xs:choice>
<!--
Element potTrailing5 is potentially trailing because the sequence
following it is a potentially trailing group.
-->
<xs:element name="potTrailing5" type="xs:string" minOccurs="0"/>
<xs:sequence dfdl:separator="*" dfdl:separatorSuppressionPolicy="trailingEmpty">
<!--
This sequence is potentially trailing,
If empty and trailing the comma for this group in the enclosing sequence may be omitted.
-->
<xs:element name="potTrailing6" type="xs:int" minOccurs="0" maxOccurs="3" />
</xs:sequence>
<xs:element name="potTrailing7" type="xs:string" minOccurs="0"/>
<xs:element name="potTrailing8" type="xs:string" minOccurs="0"/>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>

</tdml:defineSchema>

<tdml:parserTestCase name="ptg3_2p_daf" root="ptg3" model="s3"
description="Test of potentially trailing groups. Daffodil specific as it retains required empty string elements."
roundTrip="none"
Expand All @@ -653,7 +715,7 @@
<ex:ptg3>
<record>
<required1>a</required1>
<!-- empty string has no syntax, so even when dfdlx:emptyElementParsePolicy is 'treatAsEmpty'
<!-- empty string has no syntax, so even when dfdl:emptyElementParsePolicy is 'treatAsEmpty'
we still don't create optional empty string elements.
<positional2/>
<positional3/>
Expand All @@ -680,9 +742,8 @@

</tdml:parserTestCase>

<tdml:parserTestCase name="ptg3_1p" root="ptg3" model="s3"
<tdml:parserTestCase name="ptg3_1p" root="ptg3" model="s3_absent"
description="Test of potentially trailing groups."
config="cfg_noEmptyElements"
roundTrip="none">

<tdml:document><![CDATA[|a,,,,|a,,,,|]]></tdml:document>
Expand All @@ -692,12 +753,12 @@
<ex:ptg3>
<record>
<required1>a</required1>
<!-- <positional4_s/> scalar, but is backtracked anyway by IBM DFDL, daffodil treatAsMissing mode. -->
<!-- <positional4_s/> scalar, but is backtracked anyway by IBM DFDL, daffodil treatAsAbsent mode. -->
<!-- no empty string for positional5 -->
</record>
<record>
<required1>a</required1>
<!-- <positional4_s/> scalar, but is backtracked anyway by IBM DFDL, daffodil treatAsMissing mode. -->
<!-- <positional4_s/> scalar, but is backtracked anyway by IBM DFDL, daffodil treatAsAbsent mode. -->
<!-- no empty string for positional5 -->
</record>
</ex:ptg3>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@
<xs:sequence dfdl:separator="," dfdl:separatorSuppressionPolicy="anyEmpty">
<xs:element name="foo" type="xs:string" dfdl:lengthKind="delimited" />
<xs:element name="bar" type="xs:string" dfdl:lengthKind="delimited"
minOccurs="0" dfdl:occursCountKind="implicit" dfdlx:emptyElementParsePolicy="treatAsEmpty" />
minOccurs="0" dfdl:occursCountKind="implicit" dfdl:emptyElementParsePolicy="treatAsEmpty" />
</xs:sequence>
</xs:complexType>
</xs:element>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@

<tdml:defineConfig name="cfg_noEmptyElements">
<daf:tunables>
<daf:defaultEmptyElementParsePolicy>treatAsMissing</daf:defaultEmptyElementParsePolicy>
<daf:defaultEmptyElementParsePolicy>treatAsAbsent</daf:defaultEmptyElementParsePolicy>
<daf:suppressSchemaDefinitionWarnings>
unsupportedAttributeFormDefault
encodingErrorPolicyError
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>
<dfdl:format ref="ex:GeneralFormat"
lengthUnits="bits"
dfdlx:emptyElementParsePolicy="treatAsEmpty"
emptyElementParsePolicy="treatAsEmpty"
representation="binary"
/>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd"/>
<dfdl:format ref="ex:GeneralFormat"
lengthUnits="bits"
dfdlx:emptyElementParsePolicy="treatAsEmpty"
emptyElementParsePolicy="treatAsEmpty"
/>

<dfdl:defineVariable name="myVar1" type="xs:integer" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
-->
<xs:element name="nz" type="xs:string" minOccurs="0" maxOccurs="0"
dfdl:lengthPattern='[^\x{00}]+' dfdl:lengthKind="pattern"
dfdl:occursCountKind="parsed" dfdl:encoding="iso-8859-1" dfdlx:emptyElementParsePolicy="treatAsMissing">
dfdl:occursCountKind="parsed" dfdl:encoding="iso-8859-1" dfdl:emptyElementParsePolicy="treatAsAbsent">
</xs:element>
</xs:sequence>
</xs:complexType>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@
<xs:sequence dfdl:separator="$;" dfdl:separatorPosition="infix">
<xs:element name="x" type="xs:string" dfdl:escapeSchemeRef="tns:scenario2" />
<xs:element name="y" type="xs:string" minOccurs="0" dfdl:escapeSchemeRef="tns:scenario2"
dfdlx:emptyElementParsePolicy="treatAsEmpty" />
dfdl:emptyElementParsePolicy="treatAsEmpty" />
</xs:sequence>
</xs:complexType>
</xs:element>
Expand Down
Loading