From b7e8b752fa5cc511da54b4cb2b0b8efd0205be10 Mon Sep 17 00:00:00 2001 From: Mike McGann Date: Tue, 20 Dec 2022 10:04:13 -0500 Subject: [PATCH] Add emptyElementParsePolicy to main dfdl namespace. The emptyElementParsePolicy is currently implemented as a property that is an extension to DFDL. As of DFDL 1.0, this property is now available in the main schema. This property has now been added and a deprecation message is emitted when the old property is used. The property value of 'treatAsMissing' has been renamed to 'treatAsAbsent' in DFDL 1.0. DAFFODIL-2496 --- .../daffodil/dsom/DFDLFormatAnnotation.scala | 1 + .../grammar/primitives/SequenceChild.scala | 4 +- .../daffodil/xsd/DFDLGeneralFormat.dfdl.xsd | 3 +- .../annotation/props/ByHandMixins.scala | 21 ++- .../daffodil/xsd/DFDL_part1_simpletypes.xsd | 7 + .../daffodil/xsd/DFDL_part2_attributes.xsd | 8 +- .../org/apache/daffodil/xsd/dafext.xsd | 6 +- .../org/apache/daffodil/xsd/dfdlx.xsd | 6 +- .../parsers/SequenceChildBases.scala | 6 +- .../SequenceChildParseResultHelper.scala | 2 +- .../test-suite/tresys-contributed/nilled.tdml | 2 +- .../tresys-contributed/sepSuppression.tdml | 87 +++++++++-- .../tresys-contributed/sepSuppression2.tdml | 2 +- .../tresys-contributed/unseparated.tdml | 2 +- .../choiceBranchKeyRanges.tdml | 2 +- .../extensions/lookAhead/lookAhead.tdml | 2 +- .../daffodil/section05/facets/NulChars.tdml | 2 +- .../escapeScheme/escapeScenarios.tdml | 2 +- .../daffodil/usertests/MultipartBody.dfdl.xsd | 4 +- .../apache/daffodil/usertests/SepTests.tdml | 138 ++++++++++++------ .../daffodil/usertests/TestSepTests.scala | 12 +- 21 files changed, 224 insertions(+), 95 deletions(-) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/DFDLFormatAnnotation.scala b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/DFDLFormatAnnotation.scala index 78c9241fff..f159c83875 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/dsom/DFDLFormatAnnotation.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/dsom/DFDLFormatAnnotation.scala @@ -39,6 +39,7 @@ object DeprecatedProperty { DeprecatedProperty(XMLUtils.DFDL_NAMESPACE, "layerLength", "dfdlx:layerLength"), DeprecatedProperty(XMLUtils.DFDL_NAMESPACE, "layerLengthUnits", "dfdlx:layerLengthUnits"), DeprecatedProperty(XMLUtils.DFDL_NAMESPACE, "layerBoundaryMark", "dfdlx:layerBoundaryMark"), + DeprecatedProperty(XMLUtils.DFDLX_NAMESPACE, "emptyElementParsePolicy", "dfdl:emptyElementParsePolicy"), DeprecatedProperty(XMLUtils.EXT_NS_APACHE, "parseUnparsePolicy", "dfdlx:parseUnparsePolicy"), DeprecatedProperty(XMLUtils.EXT_NS_NCSA, "parseUnparsePolicy", "dfdlx:parseUnparsePolicy")) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/SequenceChild.scala b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/SequenceChild.scala index a83d664894..ee3415cf2e 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/SequenceChild.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/grammar/primitives/SequenceChild.scala @@ -438,11 +438,11 @@ class ScalarOrderedSequenceChild(sq: SequenceTermBase, term: Term, groupIndex: I /** * Must deal with nils, emptyness and string/hexBinary exceptional behavior - * including the behavior for dfdlx:emptyElementParsePolicy 'treatAsMissing' which special cases + * including the behavior for dfdl:emptyElementParsePolicy 'treatAsAbsent' which special cases * Required elements like scalars, iff they are emptyRep, emptyValueDelimiterPolicy, * nilValueDelimiterPolicy, complex elements that nillable, or fully defaultable. * - * So we have ((simpleStringHexBinary x (treatAsMissing, treatAsEmpty), simpleOther, complex) x (nillable, not) x + * So we have ((simpleStringHexBinary x (treatAsAbsent, treatAsEmpty), simpleOther, complex) x (nillable, not) x * 4 behaviors. That's 32 combinations. Let's start with fewer cases and more runtime * decisions, and specialize if we think it will help clarity or performance. */ diff --git a/daffodil-lib/src/main/resources/org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd b/daffodil-lib/src/main/resources/org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd index d460720124..af7e00de7b 100644 --- a/daffodil-lib/src/main/resources/org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd +++ b/daffodil-lib/src/main/resources/org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd @@ -103,7 +103,8 @@ - + diff --git a/daffodil-lib/src/main/scala/org/apache/daffodil/schema/annotation/props/ByHandMixins.scala b/daffodil-lib/src/main/scala/org/apache/daffodil/schema/annotation/props/ByHandMixins.scala index 93f99ea540..6a994d579f 100644 --- a/daffodil-lib/src/main/scala/org/apache/daffodil/schema/annotation/props/ByHandMixins.scala +++ b/daffodil-lib/src/main/scala/org/apache/daffodil/schema/annotation/props/ByHandMixins.scala @@ -430,9 +430,11 @@ trait TextStandardExponentRepMixin extends PropertyMixin { */ sealed trait EmptyElementParsePolicy extends EmptyElementParsePolicy.Value object EmptyElementParsePolicy extends Enum[EmptyElementParsePolicy] { - case object TreatAsMissing extends EmptyElementParsePolicy + case object TreatAsMissing extends EmptyElementParsePolicy // deprecated case object TreatAsEmpty extends EmptyElementParsePolicy - override lazy val values = Array(TreatAsMissing, TreatAsEmpty) + case object TreatAsAbsent extends EmptyElementParsePolicy + + override lazy val values = Array(TreatAsMissing, TreatAsEmpty, TreatAsAbsent) // deprecated: TreatAsMissing def apply(name: String, context: ThrowsSDE): EmptyElementParsePolicy = stringToEnum("emptyElementParsePolicy", name, context) } @@ -454,7 +456,7 @@ trait EmptyElementParsePolicyMixin extends PropertyMixin { /** * Property determines whether Daffodil implements empty elements in a manner consistent with - * IBM DFDL (as of 2019-05-02), which has policy treatAsMissing, or implements what is + * IBM DFDL (as of 2019-05-02), which has policy treatAsAbsent, or implements what is * described in the DFDL spec., which is treatAsEmpty - if the syntax of * empty (or nullness) is matched, create an empty (or null) item, even if optional, unless * the element is entirely absent. @@ -468,15 +470,10 @@ trait EmptyElementParsePolicyMixin extends PropertyMixin { // prop is not required AND not defined so use tunable value // but issue warning (which can be suppressed) val defaultEmptyElementParsePolicy = this.tunable.defaultEmptyElementParsePolicy - // This property is an extension, so we don't want to require users to - // add this property just to silence this warning, especially since the - // property might change in the future. So silently use a default without - // outputting a warning. We may want to turn this warning on when the - // property makes its way into the official DFDL spec. - //SDW( - // WarnID.EmptyElementParsePolicyError, - // "Property 'dfdlx:emptyElementParsePolicy' is required but not defined, using tunable '%s' by default.", - // defaultEmptyElementParsePolicy) + SDW( + WarnID.EmptyElementParsePolicyError, + "Property 'dfdl:emptyElementParsePolicy' is required but not defined, using tunable '%s' by default.", + defaultEmptyElementParsePolicy) defaultEmptyElementParsePolicy } } diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd index 9d6f351e31..1bc9a1df9a 100644 --- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd +++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part1_simpletypes.xsd @@ -626,6 +626,13 @@ + + + + + + + diff --git a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part2_attributes.xsd b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part2_attributes.xsd index 9f0b2a2159..9fd404dd2f 100644 --- a/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part2_attributes.xsd +++ b/daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/DFDL_part2_attributes.xsd @@ -98,8 +98,9 @@ - - + + + @@ -539,6 +540,7 @@ + @@ -641,6 +643,8 @@ type="dfdl:EncodingEnum_Or_DFDLExpression" /> + - + + + + + @@ -94,11 +94,11 @@ - + - + diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceChildBases.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceChildBases.scala index ad591f0ea8..90d6f5eb67 100644 --- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceChildBases.scala +++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceChildBases.scala @@ -74,11 +74,11 @@ object ParseAttemptStatus { * * The EmptyRep for simpleTypes enables default values to be substituted at parse time. * - * For simple types xs:string and xs:hexBinary, the property dfdlx:emptyElementParsePolicy controls + * For simple types xs:string and xs:hexBinary, the property dfdl:emptyElementParsePolicy controls * whether the EmptyRep is allowed for strings and hexBinary. In required positions, when - * dfdlx:emptyElementParsePolicy is 'treatAsMissing', a required string/hexBinary that has EmptyRep + * dfdl:emptyElementParsePolicy is 'treatAsAbsent', a required string/hexBinary that has EmptyRep * causes a Parse Error, and an optional EmptyRep causes nothing to be added to the infoset (the empty string - * or hexBinary value is suppressed). When dfdlx:emptyElementParsePolicy is 'treatAsEmpty', a required + * or hexBinary value is suppressed). When dfdl:emptyElementParsePolicy is 'treatAsEmpty', a required * string/hexBinary with EmptyRep creates an empty string or zero-length byte array in the infoset. * An optional EmptyRep behaves differently depending on whether the EmptyRep is truly zero-length, or * dfdl:emptyValueDelimiterPolicy is such that EmptyRep is non-zero-length. When truly zero-length, no diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceChildParseResultHelper.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceChildParseResultHelper.scala index 98bfa3057f..8142b27ff8 100644 --- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceChildParseResultHelper.scala +++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/processors/parsers/SequenceChildParseResultHelper.scala @@ -274,7 +274,7 @@ trait ElementSequenceChildParseResultHelper pstate.schemaDefinitionError("Default values not implemented.") } else { emptyElementParsePolicy match { - case EmptyElementParsePolicy.TreatAsMissing => { + case EmptyElementParsePolicy.TreatAsMissing | EmptyElementParsePolicy.TreatAsAbsent => { // deprecated: TreatAsMissing parser.PE(pstate, "Empty element not allowed for required element.") ParseAttemptStatus.MissingItem } diff --git a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nilled.tdml b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nilled.tdml index f3873bc8d7..e37855d429 100644 --- a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nilled.tdml +++ b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/nilled.tdml @@ -38,7 +38,7 @@ - treatAsMissing + treatAsAbsent unsupportedAttributeFormDefault encodingErrorPolicyError diff --git a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/sepSuppression.tdml b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/sepSuppression.tdml index 72f893ec9f..7bccbc5367 100644 --- a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/sepSuppression.tdml +++ b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/sepSuppression.tdml @@ -36,12 +36,12 @@ - + - treatAsMissing + treatAsAbsent - unsupportedAttributeFormDefault + unsupportedAttributeFormDefault encodingErrorPolicyError @@ -209,7 +209,7 @@ a @@ -257,7 +257,7 @@ a - @@ -283,7 +283,7 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + a - + a - + diff --git a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/sepSuppression2.tdml b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/sepSuppression2.tdml index bd9e3290c4..a75ddec036 100644 --- a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/sepSuppression2.tdml +++ b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/sepSuppression2.tdml @@ -283,7 +283,7 @@ + minOccurs="0" dfdl:occursCountKind="implicit" dfdl:emptyElementParsePolicy="treatAsEmpty" /> diff --git a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/unseparated.tdml b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/unseparated.tdml index e520ba06d5..fb4fa0ac42 100644 --- a/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/unseparated.tdml +++ b/daffodil-test-ibm1/src/test/resources/test-suite/tresys-contributed/unseparated.tdml @@ -36,7 +36,7 @@ - treatAsMissing + treatAsAbsent unsupportedAttributeFormDefault encodingErrorPolicyError diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/extensions/choiceBranchRanges/choiceBranchKeyRanges.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/extensions/choiceBranchRanges/choiceBranchKeyRanges.tdml index 1b9f2ee338..50b8dc4177 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/extensions/choiceBranchRanges/choiceBranchKeyRanges.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/extensions/choiceBranchRanges/choiceBranchKeyRanges.tdml @@ -29,7 +29,7 @@ diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/extensions/lookAhead/lookAhead.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/extensions/lookAhead/lookAhead.tdml index 30f2e6f131..351500324e 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/extensions/lookAhead/lookAhead.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/extensions/lookAhead/lookAhead.tdml @@ -28,7 +28,7 @@ diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section05/facets/NulChars.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section05/facets/NulChars.tdml index 6e2a412439..66d224e88e 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section05/facets/NulChars.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section05/facets/NulChars.tdml @@ -60,7 +60,7 @@ --> + dfdl:occursCountKind="parsed" dfdl:encoding="iso-8859-1" dfdl:emptyElementParsePolicy="treatAsAbsent"> diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeScenarios.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeScenarios.tdml index 7b3f9a356b..96143b8548 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeScenarios.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeScenarios.tdml @@ -344,7 +344,7 @@ + dfdl:emptyElementParsePolicy="treatAsEmpty" /> diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/MultipartBody.dfdl.xsd b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/MultipartBody.dfdl.xsd index 86230544fc..70f2ff7d15 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/MultipartBody.dfdl.xsd +++ b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/MultipartBody.dfdl.xsd @@ -53,8 +53,8 @@ dfdl:terminator="--%CR;%LF;"> + dfdl:occursCountKind="implicit" + dfdl:emptyElementParsePolicy="treatAsEmpty" /> diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/SepTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/SepTests.tdml index e338f16587..b2257d3378 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/usertests/SepTests.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/usertests/SepTests.tdml @@ -193,12 +193,7 @@ ref="ex:GeneralFormatPortable" representation="text" lengthKind="delimited" - separatorPosition="infix" - dfdlx:emptyElementParsePolicy="treatAsEmpty"/> - + separatorPosition="infix"/> @@ -271,25 +266,46 @@ - + + + madonna,,,,,,,,, + + + + madonna + + + + + + + madonna,,,,,,,,, + + + Parse Error + emptyElementParsePolicy + treatAsMissing + required + + + - + ref="ex:GeneralFormatPortable" + representation="text" + lengthKind="delimited" + separatorPosition="infix" + dfdlx:emptyElementParsePolicy="treatAsMissing"/> @@ -304,9 +320,7 @@ @@ -321,33 +335,73 @@ - - + madonna,,,,,,,,, - - - - madonna - - - + + emptyElementParsePolicy is deprecated + Use dfdl:emptyElementParsePolicy + + + Parse Error + Empty element not allowed + required + - - madonna,,,,,,,,, + + + + + + + treatAsAbsent should have no effect here, because everything is optional. + --> + + + + + + + + + + + + + + + + + + + + + + + + + madonna,,,,,,,,, Parse Error - emptyElementParsePolicy - treatAsMissing + Empty element not allowed required - - \ No newline at end of file + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestSepTests.scala b/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestSepTests.scala index a605680128..81ccf50e75 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestSepTests.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/usertests/TestSepTests.scala @@ -23,7 +23,7 @@ import org.junit.AfterClass object TestSepTests { val testDir = "/org/apache/daffodil/usertests/" - val runner = Runner(testDir, "SepTests.tdml") + val runner: Runner = Runner(testDir, "SepTests.tdml") @AfterClass def shutDown(): Unit = { runner.reset @@ -58,10 +58,10 @@ class TestSepTests { // Add daffodil to implementations to see the erroneous daffodil behavior. @Test def test_sep_ssp_never_3(): Unit = { runner.runOneTest("test_sep_ssp_never_3") } - // DAFFODIL-2496 - implement DFDL official emptyElementParsePolicy property - // Note: this test isn't commented out, because it works for IBM DFDL in cross testing - // The TDML for this test just has it disabled for the daffodil implementation. - // Add daffodil to implementations to see the erroneous daffodil behavior. - @Test def test_sep_ssp_never_4(): Unit = { runner.runOneTest("test_sep_ssp_never_4") } + @Test def test_sep_ssp_never_4_ibm(): Unit = { runner.runOneTest("test_sep_ssp_never_4_ibm") } + + @Test def test_sep_ssp_never_4_daffodil(): Unit = { runner.runOneTest("test_sep_ssp_never_4_daffodil") } + + @Test def test_sep_ssp_never_5(): Unit = { runner.runOneTest("test_sep_ssp_never_5") } }