add end-of-line normalization per invisibleXML/ixml#242

GuntherRademacher · Apr 5, 2024 · 945f6a0 · 945f6a0
1 parent 09abb3d
commit 945f6a0
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 6 deletions.
diff --git a/src/main/java/de/bottlecaps/markup/blitz/Parser.java b/src/main/java/de/bottlecaps/markup/blitz/Parser.java
@@ -48,6 +48,7 @@ public class Parser
   private final int[] forks;
   private final BitSet[] expectedTokens;
   private final boolean isVersionMismatch;
+  private final boolean normalizeEol;
 
   private Writer err = new OutputStreamWriter(System.err, StandardCharsets.UTF_8);
 
@@ -61,7 +62,8 @@ public Parser(
       RangeSet[] terminal,
       int[] forks,
       BitSet[] expectedTokens,
-      boolean isVersionMismatch) {
+      boolean isVersionMismatch,
+      boolean normalizeEol) {
 
     this.defaultOptions = defaultOptions;
     this.asciiMap = asciiMap;
@@ -77,6 +79,7 @@ public Parser(
     this.forks = forks;
     this.expectedTokens = expectedTokens;
     this.isVersionMismatch = isVersionMismatch;
+    this.normalizeEol = normalizeEol;
   }
 
   /**
@@ -979,6 +982,11 @@ private int match() {
             if (trace)
               if (c1 >= 32 && c1 <= 126)
                 writeTrace(" char=\"" + xmlEscape(String.valueOf((char) c1)) + "\"");
+            if (c1 == 0xD && normalizeEol) {
+              if (e1 < size && input.charAt(e1) == 0xA)
+                ++e1;
+              c1 = 0xA;
+            }
             charclass = asciiMap[c1];
           }
           else if (c1 < 0xd800) {

diff --git a/src/main/java/de/bottlecaps/markup/blitz/grammar/Grammar.java b/src/main/java/de/bottlecaps/markup/blitz/grammar/Grammar.java
@@ -51,7 +51,7 @@ private Grammar(String versionString, Version version, boolean mismatch) {
     this.versionString = versionString;
     this.rules = new LinkedHashMap<>();
     if (version != null) {
-      this.version = Version.UNSPECIFIED;
+      this.version = version;
       this.mismatch = mismatch;
     }
     else if (this.versionString == null) {

diff --git a/src/main/java/de/bottlecaps/markup/blitz/transform/Generator.java b/src/main/java/de/bottlecaps/markup/blitz/transform/Generator.java
@@ -192,7 +192,8 @@ public static Parser generate(Grammar g, Set<Option> options) {
         ci.terminal,
         ci.forks,
         expectedTokens,
-        ci.grammar.isMismatch());
+        ci.grammar.isMismatch(),
+        ci.grammar.getVersion().isAtLeast(Grammar.Version.V1_1));
   }
 
   private int[] asciiMap(CompressedMap bmpMap) {

diff --git a/src/test/java/de/bottlecaps/markup/BlitzTest.java b/src/test/java/de/bottlecaps/markup/BlitzTest.java
@@ -168,8 +168,14 @@ public void testMultiThreadParsing() throws Throwable {
 
   @Test
   public void testCrLf() {
-    Parser parser = generate("S:  ~[]*.");
-    assertEquals("<S>&#xD;\n</S>", parser.parse("\r\n"));
+    final String grammar = "S:  ~[]*.";
+    final String crLf = "\r\n";
+    Parser parser = generate(grammar);
+    assertEquals("<S>\n</S>", parser.parse(crLf));
+    parser = generate("ixml version '1.0'. " + grammar);
+    assertEquals("<S>&#xD;\n</S>", parser.parse(crLf));
+    parser = generate("ixml version '1.1'. " + grammar);
+    assertEquals("<S>\n</S>", parser.parse(crLf));
   }
 
   @Test

diff --git a/src/test/java/de/bottlecaps/markup/blitz/ixml/IxmlCommunityTest.java b/src/test/java/de/bottlecaps/markup/blitz/ixml/IxmlCommunityTest.java
@@ -87,6 +87,18 @@ private SkipReason(String detail) {
 
     // waiting for https://github.com/invisibleXML/ixml/pull/241
     skipReasons.put("Tests producing parse trees/version-decl-two", SkipReason.BROKEN);
+
+    // waiting for https://github.com/invisibleXML/ixml/pull/245
+    skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-06", SkipReason.BROKEN);
+    skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-07", SkipReason.BROKEN);
+    skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-08", SkipReason.BROKEN);
+    skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-09", SkipReason.BROKEN);
+    skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-10", SkipReason.BROKEN);
+    skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-ORP", SkipReason.BROKEN);
+    skipReasons.put("Oberon-performance-tests/Oberon-modules/ORS", SkipReason.BROKEN);
+    skipReasons.put("Oberon-performance-tests/Oberon-modules/ORB", SkipReason.BROKEN);
+    skipReasons.put("Oberon-performance-tests/Oberon-modules/ORG", SkipReason.BROKEN);
+    skipReasons.put("Oberon-performance-tests/Oberon-modules/ORP", SkipReason.BROKEN);
   }
 
   public static enum Catalog {
@@ -428,7 +440,7 @@ private void test(TestCase testCase) {
             .anyMatch(expected -> actual.equals(expected) || deepEqual(expected, actual))) {
           if (testCase.getOutputs().size() == 1) {
             String expected = testCase.getOutputs().get(0);
-            if (! expected.equals("<tbd/>\n"))
+            if (! expected.startsWith("<tbd/>"))
               assertEquals(expected, actual, "Unexpected parsing result");
           }
           else {