From 22b7bc7238a5434570ebd088357c9f25bf09a8c9 Mon Sep 17 00:00:00 2001 From: mbeckerle Date: Wed, 22 Nov 2017 13:18:00 -0500 Subject: [PATCH] Eliminate spurious SDE when using lengthKind pattern on all-text file. The isScannable computation is incorrect. For entirely textual formats in one encoding it still comes up with isScannable false. This causes a SDE if you are using lengthKind pattern. The SDE says the data must be scannable in order to use lengthKind pattern. DAFFODIL-1864 --- DISCLAIMER | 7 + .../ncsa/daffodil/dsom/GroupBase.scala | 5 +- .../daffodil/dsom/TermEncodingMixin.scala | 9 +- .../ncsa/daffodil/dsom/TestIsScannable.scala | 140 ++++++++++++++++++ .../ncsa/daffodil/processors/EvEncoding.scala | 2 +- 5 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 DISCLAIMER create mode 100644 daffodil-core/src/test/scala-new/edu/illinois/ncsa/daffodil/dsom/TestIsScannable.scala diff --git a/DISCLAIMER b/DISCLAIMER new file mode 100644 index 0000000000..7fcd42d4eb --- /dev/null +++ b/DISCLAIMER @@ -0,0 +1,7 @@ +Apache Daffodil is an effort undergoing incubation at The Apache Software +Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is required +of all newly accepted projects until a further review indicates that the +infrastructure, communications, and decision making process have stabilized in +a manner consistent with other successful ASF projects. While incubation status +is not necessarily a reflection of the completeness or stability of the code, +it does indicate that the project has yet to be fully endorsed by the ASF. \ No newline at end of file diff --git a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/GroupBase.scala b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/GroupBase.scala index 6b39c362cc..00c66976e0 100644 --- a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/GroupBase.scala +++ b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/GroupBase.scala @@ -81,7 +81,10 @@ abstract class GroupBase(xmlArg: Node, parentArg: SchemaComponent, position: Int final lazy val alignmentValueInBits: JInt = { this.alignment match { - case AlignmentType.Implicit => 1 + case AlignmentType.Implicit => this.alignmentUnits match { + case AlignmentUnits.Bits => 1 + case AlignmentUnits.Bytes => 8 + } case align: JInt => this.alignmentUnits match { case AlignmentUnits.Bits => align case AlignmentUnits.Bytes => 8 * align diff --git a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/TermEncodingMixin.scala b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/TermEncodingMixin.scala index bdb55a7b1e..d67f252e0c 100644 --- a/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/TermEncodingMixin.scala +++ b/daffodil-core/src/main/scala/edu/illinois/ncsa/daffodil/dsom/TermEncodingMixin.scala @@ -152,7 +152,7 @@ trait TermEncodingMixin extends KnownEncodingMixin { self: Term => val res = summaryEncoding match { case Mixed => false case Binary => false - case NoText => false + case NoText => true case Runtime => false case _ => true } @@ -170,14 +170,9 @@ trait TermEncodingMixin extends KnownEncodingMixin { self: Term => s2: EncodingLattice): EncodingLattice = { (s1, s2) match { case (x, y) if (x == y) => x - case (Mixed, _) => Mixed - case (_, Mixed) => Mixed - case (Binary, Binary) => Binary - case (Binary, _) => Mixed - case (_, Binary) => Mixed case (NoText, x) => x case (x, NoText) => x - case (x, y) => Mixed + case _ => Mixed } } diff --git a/daffodil-core/src/test/scala-new/edu/illinois/ncsa/daffodil/dsom/TestIsScannable.scala b/daffodil-core/src/test/scala-new/edu/illinois/ncsa/daffodil/dsom/TestIsScannable.scala new file mode 100644 index 0000000000..0f84f877ff --- /dev/null +++ b/daffodil-core/src/test/scala-new/edu/illinois/ncsa/daffodil/dsom/TestIsScannable.scala @@ -0,0 +1,140 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package edu.illinois.ncsa.daffodil.dsom + +import org.junit.Test +import edu.illinois.ncsa.daffodil.compiler._ + +import edu.illinois.ncsa.daffodil.util.Logging +import edu.illinois.ncsa.daffodil.xml.XMLUtils +import junit.framework.Assert._ +import edu.illinois.ncsa.daffodil.util.SchemaUtils +import org.junit.Test + +class TestIsScannable extends Logging { + + val xsd = XMLUtils.XSD_NAMESPACE + val dfdl = XMLUtils.dfdlAppinfoSource // XMLUtils.DFDL_NAMESPACE + val xsi = XMLUtils.XSI_NAMESPACE + val example = XMLUtils.EXAMPLE_NAMESPACE + + @Test def testIsScannableAllText1() { + val sc = SchemaUtils.dfdlTestSchema( + , + + + + + + + + + + + + + + + ) + + val sset = Compiler().compileNode(sc).sset + + val Seq(schema) = sset.schemas + val Seq(schemaDoc, _) = schema.schemaDocuments + val listDecl = schemaDoc.globalElementDecls.head + val list = listDecl.forRoot() + assertTrue(list.isScannable) + val Seq(child) = list.termChildren + assertTrue(child.isScannable) + assertEquals(NamedEncoding("UTF-8"), child.summaryEncoding) + } + + @Test def testIsScannableHasBinary1() { + val sc = SchemaUtils.dfdlTestSchema( + , + + + + + + + + + + + + + + + ) + + val sset = Compiler().compileNode(sc).sset + + val Seq(schema) = sset.schemas + val Seq(schemaDoc, _) = schema.schemaDocuments + val listDecl = schemaDoc.globalElementDecls.head + val list = listDecl.forRoot() + assertFalse(list.isScannable) + val Seq(child) = list.termChildren + assertFalse(child.isScannable) + val Seq(s1) = child.termChildren + assertFalse(s1.isScannable) + val Seq(gr1) = s1.termChildren + assertEquals(Mixed, gr1.summaryEncoding) + val Seq(e1, e2) = gr1.termChildren + assertEquals(Binary, e1.summaryEncoding) + assertEquals(NamedEncoding("US-ASCII"), e2.summaryEncoding) + } + + @Test def testIsScannableDifferentEncodings1() { + val sc = SchemaUtils.dfdlTestSchema( + , + + + + + + + + + + + + + + ) + + val sset = Compiler().compileNode(sc).sset + + val Seq(schema) = sset.schemas + val Seq(schemaDoc, _) = schema.schemaDocuments + val listDecl = schemaDoc.globalElementDecls.head + val list = listDecl.forRoot() + assertFalse(list.isScannable) + val Seq(child) = list.termChildren + assertFalse(child.isScannable) + val Seq(s1) = child.termChildren + assertFalse(s1.isScannable) + val Seq(gr1) = s1.termChildren + assertEquals(Mixed, gr1.summaryEncoding) + val Seq(e1, e2) = gr1.termChildren + assertEquals(NamedEncoding("US-ASCII"), e1.summaryEncoding) + assertEquals(NamedEncoding("UTF-8"), e2.summaryEncoding) + assertTrue(e1.isScannable) + assertTrue(e2.isScannable) + assertFalse(gr1.isScannable) + } + +} diff --git a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvEncoding.scala b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvEncoding.scala index 1e71f30946..e5f717bfcf 100644 --- a/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvEncoding.scala +++ b/daffodil-runtime1/src/main/scala/edu/illinois/ncsa/daffodil/processors/EvEncoding.scala @@ -98,7 +98,7 @@ class CharsetEv(encodingEv: EncodingEv, val trd: TermRuntimeData) val encString = encodingEv.evaluate(state) val cs = CharsetUtils.getCharset(encString) Assert.invariant(cs ne null) - new DFDLCharset(encString) + new DFDLCharset(cs.name.toUpperCase) } }