Allow invalid header element chars in Lexer, allow escape in header

xlate · Jun 11, 2020 · 31a97bf · 31a97bf
1 parent 58793d4
commit 31a97bf
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 10 deletions.
diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/Lexer.java
@@ -220,6 +220,7 @@ public void parse() throws IOException, EDIException {
                 handleStateInterchangeCandidate(input);
                 break;
             case HEADER_DATA:
+            case HEADER_INVALID_DATA:
                 handleStateHeaderData(input);
                 eventsReady = dialectConfirmed(State.TAG_SEARCH);
                 break;

diff --git a/src/main/java/io/xlate/edi/internal/stream/tokenization/State.java b/src/main/java/io/xlate/edi/internal/stream/tokenization/State.java
@@ -140,7 +140,7 @@ public enum State {
         /*-
          *                     SPACE ~   B   ~   I   ~   S   ~   Z   ~ SEGMT ~ CMPST ~ RELSE ~ CNTRL ~ INVLD *
          *                       |       |       |       |       |       |       |       |       |       |   *
-         *                       |   A   |   E   |   N   |   U   | ALNUM | ELEMT | REP_E | WHITE | OTHER |   *
+         *                       |   A   |   E   |   N   |   U   | ALNUM | ELEMT | RPEAT | WHITE | OTHER |   *
          *                       |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   |   *
          *                       | ~ | ~ | ~ | ~ | ~ | ~ | ~ | ~ | ~ | ~ | ~ |   |   |   |   |   |   |   |   */
         /* II | IE Initial  */{ II, __, __, __, HI, __, __, HU, __, __, __, __, __, __, __, II, II, __, __ },
@@ -150,7 +150,7 @@ public enum State {
         /* ISA / S       -> */{ __, IC, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __ },
         /* UNB / U       -> */{ __, __, __, __, __, HN, __, __, __, __, __, __, __, __, __, __, __, __, __ },
 
-        /* IC | HD          */{ HD, HD, HD, HD, HD, HD, HD, HD, HD, HD, HZ, HE, HC, __, __, HD, HD, HD, HV },
+        /* IC | HD          */{ HD, HD, HD, HD, HD, HD, HD, HD, HD, HD, HZ, HE, HC, __, DR, HD, HD, HD, HV },
 
         /* B0  Header Search*/{ B0, __, __, __, __, __, __, B1, __, __, __, __, __, __, __, B0, __, __, __ },
         /* B1  UNB (U)      */{ __, __, __, __, __, B2, __, __, __, __, __, __, __, __, __, __, __, __, __ },

diff --git a/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java b/src/test/java/io/xlate/edi/internal/stream/StaEDIStreamWriterTest.java
@@ -675,19 +675,25 @@ public int read() throws IOException {
     void testInputEquivalenceEDIFACTA() throws Exception {
         EDIInputFactory inputFactory = EDIInputFactory.newFactory();
         final ByteArrayOutputStream expected = new ByteArrayOutputStream(16384);
+        final InputStream delegate = getClass().getResourceAsStream("/EDIFACT/invoic_d97b_una.edi");
+        final char segTerm = '~';
+        final char escapeChar = '?';
 
         InputStream source = new InputStream() {
-            final InputStream delegate;
-            {
-                delegate = getClass().getResourceAsStream("/EDIFACT/invoic_d97b_una.edi");
-            }
+            boolean unaComplete = false;
 
             @Override
             public int read() throws IOException {
                 int value = delegate.read();
 
                 if (value != -1) {
-                    expected.write(value);
+                    if (value == segTerm) {
+                        unaComplete = true;
+                    }
+                    if (value != escapeChar || !unaComplete) {
+                        // Filter the escape character after the UNA
+                        expected.write(value);
+                    }
                     return value;
                 }
 

diff --git a/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java b/src/test/java/io/xlate/edi/internal/stream/tokenization/LexerTest.java
@@ -324,11 +324,16 @@ void testRejectedX12Dialect() {
     }
 
     @Test
-    void testInvalidCharacter() {
-        InputStream stream = new ByteArrayInputStream("ISA*00*\u0008         *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~".getBytes());
+    void testInvalidCharacter() throws Exception {
+        InputStream stream = new ByteArrayInputStream((""
+                + "ISA*00*          *00*          *ZZ*ReceiverID     *ZZ*Sender         *050812*1953*^*00501*508121953*0*P*:~"
+                + "TA\u0008").getBytes()); // Backspace char in segment tag
         TestLexerEventHandler eventHandler = new TestLexerEventHandler();
         final StaEDIStreamLocation location = new StaEDIStreamLocation();
         final Lexer lexer = new Lexer(stream, StandardCharsets.UTF_8, eventHandler, location);
+        for (int i = 0; i < 19; i++) {
+            lexer.parse(); // Interchange start through end of ISA
+        }
         EDIException thrown = assertThrows(EDIException.class, lexer::parse);
         assertTrue(thrown.getMessage().contains("EDIE004"));
     }

diff --git a/src/test/resources/EDIFACT/invoic_d97b_una.edi b/src/test/resources/EDIFACT/invoic_d97b_una.edi
@@ -1,5 +1,5 @@
 UNA=*.?^~
-UNB*UNOA=3*005435656=1*006415160=1*060515=1434*00000000000778~
+UNB*UNOA=3*005435656=1*006?415160=1*060515=1434*00000000000778~
 UNH*00000000000117*INVOIC=D=97B=UN~
 BGM*380*342459*9~
 DTM*3=20060515=102~