Merge pull request #15 from xlate/13_empty_segment_validation

Fix EDIFACT UNA validation, empty segment validation
xlate · May 5, 2020 · 6ad5628 · 6ad5628
2 parents 59055c8 + 533c7af
commit 6ad5628
Show file tree

Hide file tree

Showing 13 changed files with 230 additions and 14 deletions.
diff --git a/src/main/java/io/xlate/edi/internal/schema/SchemaReaderBase.java b/src/main/java/io/xlate/edi/internal/schema/SchemaReaderBase.java
@@ -353,7 +353,7 @@ StructureType readComplexType(XMLStreamReader reader,
         final List<EDIReference> refs = new ArrayList<>(8);
         final List<EDISyntaxRule> rules = new ArrayList<>(2);
 
-        reader.nextTag();
+        readDescription(reader);
         requireElementStart(qnSequence, reader);
         readReferences(reader, types, refs);
 
@@ -579,7 +579,11 @@ void requireEvent(int eventId, XMLStreamReader reader) {
     }
 
     void requireElementStart(QName element, XMLStreamReader reader) {
-        requireEvent(XMLStreamConstants.START_ELEMENT, reader);
+        Integer event = reader.getEventType();
+
+        if (event != XMLStreamConstants.START_ELEMENT) {
+            throw schemaException("Expected XML element [" + element + "] not found", reader);
+        }
 
         if (!element.equals(reader.getName())) {
             throw schemaException("Unexpected XML element [" + reader.getName() + "]", reader);

diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/Dialect.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/Dialect.java
@@ -29,6 +29,8 @@ public interface Dialect {
 
     boolean isRejected();
 
+    boolean isServiceAdviceSegment(String tag);
+
     boolean appendHeader(CharacterSet characters, char value);
 
     char getSegmentTerminator();

diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/EDIFACTDialect.java
@@ -19,6 +19,7 @@
 
 public class EDIFACTDialect implements Dialect {
 
+    private static final String UNA = "UNA";
     private static final String UNB = "UNB";
 
     private static final String[] EMPTY = new String[0];
@@ -129,6 +130,11 @@ public boolean isRejected() {
         return rejected;
     }
 
+    @Override
+    public boolean isServiceAdviceSegment(String tag) {
+        return UNA.equals(tag);
+    }
+
     @Override
     public String getStandard() {
         return Standards.EDIFACT;

diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/ProxyEventHandler.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/ProxyEventHandler.java
@@ -172,7 +172,7 @@ public boolean segmentBegin(String segmentTag) {
         Validator validator = validator();
         boolean eventsReady = true;
 
-        if (validator != null) {
+        if (validator != null && !dialect.isServiceAdviceSegment(segmentTag)) {
             validator.validateSegment(this, segmentTag);
             eventsReady = !validator.isPendingDiscrimination();
         }

diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/X12Dialect.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/X12Dialect.java
@@ -110,6 +110,11 @@ public boolean isRejected() {
         return rejected;
     }
 
+    @Override
+    public boolean isServiceAdviceSegment(String tag) {
+        return false; // X12 does not use a service advice string
+    }
+
     @Override
     public boolean appendHeader(CharacterSet characters, char value) {
         index++;

diff --git a/src/main/java/io/xlate/edi/internal/stream/validation/Validator.java b/src/main/java/io/xlate/edi/internal/stream/validation/Validator.java
@@ -898,7 +898,8 @@ public void validateSyntax(ElementDataHandler handler, ValidationEventHandler va
         final int componentIndex = location.getComponentPosition() - 1;
         final List<UsageNode> children = structure.getChildren();
 
-        for (int i = index, max = children.size(); i < max; i++) {
+        // Ensure the start index is at least zero. Index may be -1 for empty segments
+        for (int i = Math.max(index, 0), max = children.size(); i < max; i++) {
             if (isComposite) {
                 location.incrementComponentPosition();
             } else {

diff --git a/src/main/resources/schema/EDISchema-v3.xsd b/src/main/resources/schema/EDISchema-v3.xsd
@@ -214,7 +214,7 @@
             </annotation>
             <complexType>
               <sequence>
-                <choice maxOccurs="unbounded">
+                <choice minOccurs="0" maxOccurs="unbounded">
                   <element name="element" type="tns:elementImpl" />
                   <element name="composite" type="tns:compositeImpl" />
                 </choice>
@@ -540,7 +540,7 @@
                 </documentation>
               </annotation>
               <complexType>
-                <choice maxOccurs="unbounded">
+                <choice minOccurs="0" maxOccurs="unbounded">
                   <element name="element" type="tns:elementStandard" />
                   <element name="composite" type="tns:compositeStandard" />
                 </choice>

diff --git a/src/test/java/io/xlate/edi/internal/stream/ErrorEventsTest.java b/src/test/java/io/xlate/edi/internal/stream/ErrorEventsTest.java
@@ -21,6 +21,7 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
+import java.net.URL;
 
 import org.junit.jupiter.api.Test;
 
@@ -299,4 +300,77 @@ public void testTooManySimpleElements() throws EDIStreamException {
 
         assertTrue(!reader.hasNext(), "Unexpected errors exist");
     }
+
+    @Test
+    public void testValidEmptySegment() throws EDISchemaException, EDIStreamException {
+        EDIInputFactory factory = EDIInputFactory.newFactory();
+        InputStream stream = new ByteArrayInputStream((""
+                + "ISA*00*          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~"
+                + "S01*X~"
+                + "ETY~"
+                + "S11*X~"
+                + "S12*X~"
+                + "S19*X~"
+                + "S09*X~"
+                + "IEA*1*508121953~").getBytes());
+
+        SchemaFactory schemaFactory = SchemaFactory.newFactory();
+        URL schemaLocation = getClass().getResource("/x12/EDISchemaSegmentValidation.xml");
+        Schema schema = schemaFactory.createSchema(schemaLocation);
+
+        EDIStreamReader reader = factory.createEDIStreamReader(stream, schema);
+        reader = factory.createFilteredReader(reader, (r) -> {
+            switch (r.getEventType()) {
+            case SEGMENT_ERROR:
+            case ELEMENT_DATA_ERROR:
+            case ELEMENT_OCCURRENCE_ERROR:
+            case START_TRANSACTION:
+                return true;
+            default:
+                break;
+            }
+            return false;
+        });
+
+        assertEquals(EDIStreamEvent.START_TRANSACTION, reader.next(), "Expecting start of transaction");
+        reader.setTransactionSchema(schemaFactory.createSchema(getClass().getResource("/x12/EDISchemaSegmentValidationTx.xml")));
+        assertTrue(!reader.hasNext(), "Unexpected errors exist");
+    }
+
+    @Test
+    public void testEmptySegmentSchemaWithData() throws EDISchemaException, EDIStreamException {
+        EDIInputFactory factory = EDIInputFactory.newFactory();
+        InputStream stream = new ByteArrayInputStream((""
+                + "ISA*00*          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~"
+                + "S01*X~"
+                + "ETY*DATA_SHOULD_NOT_BE_HERE~"
+                + "S11*X~"
+                + "S12*X~"
+                + "S19*X~"
+                + "S09*X~"
+                + "IEA*1*508121953~").getBytes());
+
+        SchemaFactory schemaFactory = SchemaFactory.newFactory();
+        URL schemaLocation = getClass().getResource("/x12/EDISchemaSegmentValidation.xml");
+        Schema schema = schemaFactory.createSchema(schemaLocation);
+
+        EDIStreamReader reader = factory.createEDIStreamReader(stream, schema);
+        reader = factory.createFilteredReader(reader, (r) -> {
+            switch (r.getEventType()) {
+            case SEGMENT_ERROR:
+            case ELEMENT_DATA_ERROR:
+            case ELEMENT_OCCURRENCE_ERROR:
+            case START_TRANSACTION:
+                return true;
+            default:
+                break;
+            }
+            return false;
+        });
+
+        assertEquals(EDIStreamEvent.START_TRANSACTION, reader.next(), "Expecting start of transaction");
+        reader.setTransactionSchema(schemaFactory.createSchema(getClass().getResource("/x12/EDISchemaSegmentValidationTx.xml")));
+        assertTrue(reader.hasNext(), "Expected error missing");
+        assertEquals(EDIStreamValidationError.TOO_MANY_DATA_ELEMENTS, reader.getErrorType());
+    }
 }
diff --git a/src/test/java/io/xlate/edi/internal/stream/SegmentValidationTest.java b/src/test/java/io/xlate/edi/internal/stream/SegmentValidationTest.java
@@ -303,15 +303,15 @@ public boolean accept(EDIStreamReader reader) {
         reader.setTransactionSchema(schemaFactory.createSchema(getClass().getResource("/x12/EDISchemaSegmentValidationTx.xml")));
 
         assertEquals(EDIStreamEvent.START_LOOP, reader.next());
-        assertEquals("0000", reader.getReferenceCode());
+        assertEquals("L0000", reader.getReferenceCode());
         assertEquals(EDIStreamEvent.END_LOOP, reader.next());
-        assertEquals("0000", reader.getReferenceCode());
+        assertEquals("L0000", reader.getReferenceCode());
 
         for (int i = 0; i < 2; i++) {
             assertEquals(EDIStreamEvent.START_LOOP, reader.next());
-            assertEquals("0001", reader.getReferenceCode());
+            assertEquals("L0001", reader.getReferenceCode());
             assertEquals(EDIStreamEvent.END_LOOP, reader.next());
-            assertEquals("0001", reader.getReferenceCode());
+            assertEquals("L0001", reader.getReferenceCode());
         }
 
         assertTrue(!reader.hasNext(), "Unexpected segment errors exist");

diff --git a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamReaderTest.java
@@ -939,4 +939,79 @@ public void testGetBinaryDataValid()
             reader.close();
         }
     }
+
+    @Test
+    public void testEmptySegmentValidation() throws Exception {
+
+        EDIInputFactory factory = EDIInputFactory.newFactory();
+        Schema transSchema = SchemaFactory.newFactory()
+                                          .createSchema(getClass().getResourceAsStream("/EDIFACT/empty-segment-schema.xml"));
+        factory.setProperty(EDIInputFactory.EDI_VALIDATE_CONTROL_STRUCTURE, true);
+
+        EDIStreamReader reader = factory.createEDIStreamReader(getClass().getResourceAsStream("/EDIFACT/empty-segment-example.edi"));
+        String segmentName = null;
+
+        try {
+            while (reader.hasNext()) {
+                switch (reader.next()) {
+                case START_TRANSACTION:
+                    reader.setTransactionSchema(transSchema);
+                    break;
+
+                case START_SEGMENT:
+                    segmentName = reader.getText();
+                    break;
+
+                case END_SEGMENT:
+                    segmentName = null;
+                    break;
+
+                case ELEMENT_DATA:
+                    break;
+
+                case SEGMENT_ERROR: {
+                    Location loc = reader.getLocation();
+                    EDIStreamValidationError error = reader.getErrorType();
+                    fail(String.format("%s: %s (seg=%s)",
+                                       error.getCategory(),
+                                       error,
+                                       segmentName));
+                    break;
+                }
+                case ELEMENT_DATA_ERROR:
+                    // TODO Change "control schema", because it does not recognise "IATA" (UNB), "PNRGOV:11" (UNH)
+                    if ("DE0001".equals(reader.getReferenceCode()) && "IATA".equals(reader.getText())) {
+                        break;
+                    }
+                    if ("DE0065".equals(reader.getReferenceCode()) && "PNRGOV".equals(reader.getText())) {
+                        break;
+                    }
+                    if ("DE0052".equals(reader.getReferenceCode())) {
+                        break;
+                    }
+
+                    break;
+
+                case ELEMENT_OCCURRENCE_ERROR: {
+                    Location loc = reader.getLocation();
+                    EDIStreamValidationError error = reader.getErrorType();
+
+                    // FIXME when the empty segment is reached, TOO_MANY_DATA_ELEMENTS is returned
+                    fail(String.format("%s: %s (seg=%s, elemPos=%d, compoPos=%d, textOnError=%s)",
+                                       error.getCategory(),
+                                       error,
+                                       segmentName,
+                                       loc.getElementPosition(),
+                                       loc.getComponentPosition(),
+                                       reader.getText()));
+                    break;
+                }
+                default:
+                    break;
+                }
+            }
+        } finally {
+            reader.close();
+        }
+    }
 }
diff --git a/src/test/resources/EDIFACT/empty-segment-example.edi b/src/test/resources/EDIFACT/empty-segment-example.edi
@@ -0,0 +1,8 @@
+UNA:+.\*'
+UNB+IATA:1+1A+KRC+130527:0649+0003'
+UNH+1+PNRGOV:11:1:IA+270513/0649/SQ/602'
+EQN+1'
+SRC'
+FOO+BAR'
+UNT+85+1'
+UNZ+1+0003'
diff --git a/src/test/resources/EDIFACT/empty-segment-schema.xml b/src/test/resources/EDIFACT/empty-segment-schema.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+
+<schema xmlns="http://xlate.io/EDISchema/v3">
+
+    <transaction>
+        <sequence>
+            <segment type="EQN" minOccurs="1" maxOccurs="1" />
+            <segment type="SRC" minOccurs="1" maxOccurs="1" />
+            <segment type="FOO" minOccurs="1" maxOccurs="1" />
+        </sequence>
+    </transaction>
+
+    <!-- ==================================== Element types ===================================== -->
+
+    <elementType name="Generic" base="string" minLength="0" maxLength="500" />
+    <elementType name="Number1to3" base="numeric" minLength="1" maxLength="3" />
+
+    <!-- ==================================== Segment types ===================================== -->
+
+    <segmentType name="EQN">
+        <sequence>
+            <element type="Number1to3" minOccurs="1" maxOccurs="1" />
+        </sequence>
+    </segmentType>
+
+    <segmentType name="SRC">
+        <sequence/>
+    </segmentType>
+
+    <segmentType name="FOO">
+        <sequence>
+            <element type="Generic" minOccurs="1" maxOccurs="1" />
+        </sequence>
+    </segmentType>
+
+</schema>
diff --git a/src/test/resources/x12/EDISchemaSegmentValidationTx.xml b/src/test/resources/x12/EDISchemaSegmentValidationTx.xml
@@ -14,13 +14,13 @@
   License for the specific language governing permissions and limitations under
   the License.
 -->
-<schema xmlns="http://xlate.io/EDISchema/v2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://xlate.io/EDISchema/v2 ../../../../src/main/resources/schema/EDISchema-v2.xsd">
+<schema xmlns="http://xlate.io/EDISchema/v3">
 
   <transaction>
     <sequence>
       <segment ref="S0A" />
-      <loop code="0000">
+      <segment ref="ETY" />
+      <loop code="L0000">
         <sequence>
           <segment ref="S11" minOccurs="1" />
           <segment ref="S12" minOccurs="1" />
@@ -29,7 +29,7 @@
           <segment ref="S19" minOccurs="1" />
         </sequence>
       </loop>
-      <loop code="0001" maxOccurs="5">
+      <loop code="L0001" maxOccurs="5">
         <sequence>
           <segment ref="S20" minOccurs="1" />
           <segment ref="S21" />
@@ -99,4 +99,9 @@
       <element ref="E999" minOccurs="1" />
     </sequence>
   </segmentType>
+
+  <segmentType name="ETY">
+    <description>An EmpTY segment</description>
+    <sequence />
+  </segmentType>
 </schema>