This repository has been archived by the owner on Apr 3, 2018. It is now read-only.
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
LANG: refactor SimpleLexingHelper, enable stream use.
- Loading branch information
1 parent
658f1e4
commit e3f18ce
Showing
10 changed files
with
435 additions
and
81 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 0 additions & 74 deletions
74
plugin_tooling/src-lang/melnorme/lang/utils/SimpleLexingHelper.java
This file was deleted.
Oops, something went wrong.
60 changes: 60 additions & 0 deletions
60
plugin_tooling/src-lang/melnorme/lang/utils/parse/IParseSource.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
/******************************************************************************* | ||
* Copyright (c) 2015, 2015 IBM Corporation and others. | ||
* All rights reserved. This program and the accompanying materials | ||
* are made available under the terms of the Eclipse Public License v1.0 | ||
* which accompanies this distribution, and is available at | ||
* http://www.eclipse.org/legal/epl-v10.html | ||
* | ||
* Contributors: | ||
* Bruno Medeiros - initial API and implementation | ||
*******************************************************************************/ | ||
package melnorme.lang.utils.parse; | ||
|
||
import static melnorme.utilbox.core.Assert.AssertNamespace.assertTrue; | ||
|
||
import java.io.IOException; | ||
|
||
|
||
public interface IParseSource { | ||
|
||
int consume() throws IOException; | ||
|
||
int lookahead(int n) throws IOException; | ||
|
||
/** @return number of characters available to consume without having to wait for a stream input, | ||
* and with no risk of IOException if consume() is called. | ||
* | ||
* Use intended for tests or performance optimizations */ | ||
int bufferedCharCount(); | ||
|
||
|
||
default int lookahead() throws IOException { | ||
return lookahead(0); | ||
} | ||
|
||
default boolean hasCharAhead() throws IOException { | ||
return lookahead() != -1; | ||
} | ||
|
||
default char consumeNonEOF() throws IOException { | ||
int ch = consume(); | ||
assertTrue(ch != -1); | ||
return (char) ch; | ||
} | ||
|
||
default void consume(int amount) throws IOException { | ||
while(amount-- > 0) { | ||
consume(); | ||
} | ||
} | ||
|
||
default boolean lookaheadMatches(String string) throws IOException { | ||
for(int ix = 0; ix < string.length(); ix++) { | ||
if(lookahead(ix) != string.charAt(ix)) { | ||
return false; | ||
} | ||
} | ||
return true; | ||
} | ||
|
||
} |
77 changes: 77 additions & 0 deletions
77
plugin_tooling/src-lang/melnorme/lang/utils/parse/ParseSourceUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,77 @@ | ||
/******************************************************************************* | ||
* Copyright (c) 2015, 2015 Bruno Medeiros and other Contributors. | ||
* All rights reserved. This program and the accompanying materials | ||
* are made available under the terms of the Eclipse Public License v1.0 | ||
* which accompanies this distribution, and is available at | ||
* http://www.eclipse.org/legal/epl-v10.html | ||
* | ||
* Contributors: | ||
* Bruno Medeiros - initial API and implementation | ||
*******************************************************************************/ | ||
package melnorme.lang.utils.parse; | ||
|
||
import java.io.IOException; | ||
|
||
|
||
public interface ParseSourceUtils extends IParseSource { | ||
|
||
default boolean lookaheadIsEOF() throws IOException { | ||
return !hasCharAhead(); | ||
} | ||
|
||
default boolean tryConsume(String string) throws IOException { | ||
if(lookaheadMatches(string)) { | ||
consume(string.length()); | ||
return true; | ||
} | ||
return false; | ||
} | ||
|
||
default String consumeUntil(String string) throws IOException { | ||
StringBuilder sb = new StringBuilder(); | ||
|
||
while(hasCharAhead() && !lookaheadMatches(string)) { | ||
sb.append(consumeNonEOF()); | ||
} | ||
return sb.toString(); | ||
} | ||
|
||
default String consumeUntil(String endString, boolean consumeEndString) throws IOException { | ||
String firstString = consumeUntil(endString); | ||
if(consumeEndString) { | ||
tryConsume(endString); | ||
} | ||
return firstString; | ||
} | ||
|
||
/** | ||
* Consume a string delimited by give delimiter char, | ||
* with given escapeChar acting a possible escape (use -1 for no escapeChar) | ||
*/ | ||
default String consumeDelimitedString(int delimiter, int escapeChar) throws IOException { | ||
StringBuilder sb = new StringBuilder(); | ||
|
||
while(hasCharAhead()) { | ||
|
||
char consumedChar = consumeNonEOF(); | ||
|
||
if(consumedChar == delimiter) { | ||
break; | ||
} | ||
else if(consumedChar == escapeChar && hasCharAhead()) { | ||
int lookahead = lookahead(); | ||
if(lookahead == delimiter || lookahead == escapeChar) { | ||
|
||
char escapedChar = consumeNonEOF(); | ||
sb.append(escapedChar); | ||
continue; | ||
} | ||
} | ||
|
||
sb.append(consumedChar); | ||
|
||
} | ||
return sb.toString(); | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
109 changes: 109 additions & 0 deletions
109
plugin_tooling/src-lang/melnorme/lang/utils/parse/ParseSource_Test.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
/******************************************************************************* | ||
* Copyright (c) 2015, 2015 Bruno Medeiros and other Contributors. | ||
* All rights reserved. This program and the accompanying materials | ||
* are made available under the terms of the Eclipse Public License v1.0 | ||
* which accompanies this distribution, and is available at | ||
* http://www.eclipse.org/legal/epl-v10.html | ||
* | ||
* Contributors: | ||
* Bruno Medeiros - initial API and implementation | ||
*******************************************************************************/ | ||
package melnorme.lang.utils.parse; | ||
|
||
import static melnorme.utilbox.core.Assert.AssertNamespace.assertTrue; | ||
|
||
import java.io.IOException; | ||
import java.io.StringReader; | ||
|
||
import melnorme.lang.tests.CommonToolingTest; | ||
|
||
import org.junit.Test; | ||
|
||
public class ParseSource_Test extends CommonToolingTest { | ||
|
||
protected final String TEST_SOURCE = "abcdef"; | ||
|
||
protected String source; | ||
protected int sourceIx; | ||
protected int lookahead; | ||
|
||
|
||
protected void doTest______(String source, IParseSource parseSource) throws IOException { | ||
|
||
this.source = source; | ||
this.sourceIx = 0; | ||
|
||
// Test lookahead | ||
checkBufferedCount(parseSource, 0); | ||
lookahead = testLookahead(parseSource, source.charAt(sourceIx)); | ||
|
||
// Test lookahead(1) | ||
assertTrue(parseSource.lookahead(1) == source.charAt(sourceIx + 1)); | ||
|
||
// Test consume with buffered | ||
checkBufferedCount(parseSource, 2); | ||
assertTrue(parseSource.consume() == lookahead); sourceIx++; | ||
checkBufferedCount(parseSource, 1); | ||
assertTrue(parseSource.consume() == source.charAt(sourceIx)); sourceIx++; | ||
checkBufferedCount(parseSource, 0); | ||
|
||
|
||
checkBufferedCount(parseSource, 0); | ||
lookahead = testLookahead(parseSource, source.charAt(sourceIx)); | ||
// Test consume with buffered | ||
checkBufferedCount(parseSource, 1); | ||
assertTrue(lookahead == parseSource.consume()); sourceIx++; | ||
|
||
assertTrue(lookahead == 'c'); | ||
|
||
|
||
checkBufferedCount(parseSource, 0); | ||
assertTrue(parseSource.consume() == source.charAt(sourceIx)); | ||
sourceIx++; | ||
|
||
while(sourceIx < source.length()) { | ||
int ch = testLookahead(parseSource, source.charAt(sourceIx)); | ||
assertTrue(parseSource.consume() == ch); | ||
sourceIx++; | ||
} | ||
|
||
// EOF | ||
testLookahead(parseSource, -1); | ||
assertTrue(parseSource.consume() == -1); | ||
} | ||
|
||
protected void checkBufferedCount(IParseSource parseSource, int expected) { | ||
assertTrue(parseSource.bufferedCharCount() == expected); | ||
} | ||
|
||
protected int testLookahead(IParseSource parseSource, int expected) throws IOException { | ||
int lookahead = parseSource.lookahead(); | ||
assertTrue(lookahead == expected); | ||
assertTrue(lookahead == parseSource.lookahead(0)); | ||
return lookahead; | ||
} | ||
|
||
public static class ReaderParseSource_Test extends ParseSource_Test { | ||
|
||
@Test | ||
public void test() throws Exception { test$(); } | ||
public void test$() throws Exception { | ||
doTest______(TEST_SOURCE, new ReaderParseSource(new StringReader(TEST_SOURCE))); | ||
} | ||
} | ||
|
||
public static class StringParseSource_Test extends ParseSource_Test { | ||
|
||
@Test | ||
public void test() throws Exception { test$(); } | ||
public void test$() throws Exception { | ||
doTest______(TEST_SOURCE, new StringParseSource(TEST_SOURCE)); | ||
} | ||
|
||
@Override | ||
protected void checkBufferedCount(IParseSource parseSource, int expected) { | ||
assertTrue(parseSource.bufferedCharCount() == source.length() - sourceIx); | ||
} | ||
} | ||
|
||
} |
Oops, something went wrong.