Skip to content

Commit

Permalink
add end-of-line normalization per invisibleXML/ixml#242
Browse files Browse the repository at this point in the history
  • Loading branch information
GuntherRademacher committed Apr 5, 2024
1 parent 09abb3d commit 945f6a0
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 6 deletions.
10 changes: 9 additions & 1 deletion src/main/java/de/bottlecaps/markup/blitz/Parser.java
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public class Parser
private final int[] forks;
private final BitSet[] expectedTokens;
private final boolean isVersionMismatch;
private final boolean normalizeEol;

private Writer err = new OutputStreamWriter(System.err, StandardCharsets.UTF_8);

Expand All @@ -61,7 +62,8 @@ public Parser(
RangeSet[] terminal,
int[] forks,
BitSet[] expectedTokens,
boolean isVersionMismatch) {
boolean isVersionMismatch,
boolean normalizeEol) {

this.defaultOptions = defaultOptions;
this.asciiMap = asciiMap;
Expand All @@ -77,6 +79,7 @@ public Parser(
this.forks = forks;
this.expectedTokens = expectedTokens;
this.isVersionMismatch = isVersionMismatch;
this.normalizeEol = normalizeEol;
}

/**
Expand Down Expand Up @@ -979,6 +982,11 @@ private int match() {
if (trace)
if (c1 >= 32 && c1 <= 126)
writeTrace(" char=\"" + xmlEscape(String.valueOf((char) c1)) + "\"");
if (c1 == 0xD && normalizeEol) {
if (e1 < size && input.charAt(e1) == 0xA)
++e1;
c1 = 0xA;
}
charclass = asciiMap[c1];
}
else if (c1 < 0xd800) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ private Grammar(String versionString, Version version, boolean mismatch) {
this.versionString = versionString;
this.rules = new LinkedHashMap<>();
if (version != null) {
this.version = Version.UNSPECIFIED;
this.version = version;
this.mismatch = mismatch;
}
else if (this.versionString == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,8 @@ public static Parser generate(Grammar g, Set<Option> options) {
ci.terminal,
ci.forks,
expectedTokens,
ci.grammar.isMismatch());
ci.grammar.isMismatch(),
ci.grammar.getVersion().isAtLeast(Grammar.Version.V1_1));
}

private int[] asciiMap(CompressedMap bmpMap) {
Expand Down
10 changes: 8 additions & 2 deletions src/test/java/de/bottlecaps/markup/BlitzTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,14 @@ public void testMultiThreadParsing() throws Throwable {

@Test
public void testCrLf() {
Parser parser = generate("S: ~[]*.");
assertEquals("<S>&#xD;\n</S>", parser.parse("\r\n"));
final String grammar = "S: ~[]*.";
final String crLf = "\r\n";
Parser parser = generate(grammar);
assertEquals("<S>\n</S>", parser.parse(crLf));
parser = generate("ixml version '1.0'. " + grammar);
assertEquals("<S>&#xD;\n</S>", parser.parse(crLf));
parser = generate("ixml version '1.1'. " + grammar);
assertEquals("<S>\n</S>", parser.parse(crLf));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,18 @@ private SkipReason(String detail) {

// waiting for https://github.com/invisibleXML/ixml/pull/241
skipReasons.put("Tests producing parse trees/version-decl-two", SkipReason.BROKEN);

// waiting for https://github.com/invisibleXML/ixml/pull/245
skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-06", SkipReason.BROKEN);
skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-07", SkipReason.BROKEN);
skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-08", SkipReason.BROKEN);
skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-09", SkipReason.BROKEN);
skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-10", SkipReason.BROKEN);
skipReasons.put("Oberon-performance-tests/Oberon-fragments/ob-ORP", SkipReason.BROKEN);
skipReasons.put("Oberon-performance-tests/Oberon-modules/ORS", SkipReason.BROKEN);
skipReasons.put("Oberon-performance-tests/Oberon-modules/ORB", SkipReason.BROKEN);
skipReasons.put("Oberon-performance-tests/Oberon-modules/ORG", SkipReason.BROKEN);
skipReasons.put("Oberon-performance-tests/Oberon-modules/ORP", SkipReason.BROKEN);
}

public static enum Catalog {
Expand Down Expand Up @@ -428,7 +440,7 @@ private void test(TestCase testCase) {
.anyMatch(expected -> actual.equals(expected) || deepEqual(expected, actual))) {
if (testCase.getOutputs().size() == 1) {
String expected = testCase.getOutputs().get(0);
if (! expected.equals("<tbd/>\n"))
if (! expected.startsWith("<tbd/>"))
assertEquals(expected, actual, "Unexpected parsing result");
}
else {
Expand Down

0 comments on commit 945f6a0

Please sign in to comment.