Skip to content
This repository has been archived by the owner on Apr 3, 2018. It is now read-only.

Commit

Permalink
Browse files Browse the repository at this point in the history
LANG: refactor SimpleLexingHelper, enable stream use.
  • Loading branch information
bruno-medeiros committed Jun 9, 2015
1 parent 658f1e4 commit e3f18ce
Show file tree
Hide file tree
Showing 10 changed files with 435 additions and 81 deletions.
Expand Up @@ -18,7 +18,7 @@
import melnorme.lang.ide.core.ISourceFile;
import melnorme.lang.ide.core.text.DocumentModification;
import melnorme.lang.ide.ui.text.util.AutoEditUtils;
import melnorme.lang.utils.SimpleLexingHelper;
import melnorme.lang.utils.parse.StringParserHelper;
import melnorme.utilbox.collections.ArrayList2;

import org.eclipse.jface.text.BadLocationException;
Expand Down Expand Up @@ -89,7 +89,7 @@ protected String fixIndentation(String docString) throws BadLocationException {

StringBuilder newContents = new StringBuilder();

SimpleLexingHelper parser = new SimpleLexingHelper(docString);
StringParserHelper parser = new StringParserHelper(docString);
String start = parser.consumeUntil(delimeter);
newContents.append(start);

Expand Down Expand Up @@ -119,7 +119,7 @@ protected void simpleIndent(IDocument document) throws BadLocationException {

ArrayList2<DocumentModification> changes = new ArrayList2<>();

SimpleLexingHelper parser = new SimpleLexingHelper(document.get());
StringParserHelper parser = new StringParserHelper(document.get());
parser.consumeUntil(delimeter);
while(true) {
if(!parser.tryConsume(delimeter)) {
Expand Down
1 change: 1 addition & 0 deletions plugin_tooling/META-INF/MANIFEST.MF
Expand Up @@ -20,6 +20,7 @@ Export-Package: melnorme.lang.tests,
melnorme.lang.tooling.parser.lexer,
melnorme.lang.tooling.structure,
melnorme.lang.utils,
melnorme.lang.utils.parse,
melnorme.utilbox.collections,
melnorme.utilbox.collections.iter,
melnorme.utilbox.concurrency,
Expand Down

This file was deleted.

@@ -0,0 +1,60 @@
/*******************************************************************************
* Copyright (c) 2015, 2015 IBM Corporation and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Bruno Medeiros - initial API and implementation
*******************************************************************************/
package melnorme.lang.utils.parse;

import static melnorme.utilbox.core.Assert.AssertNamespace.assertTrue;

import java.io.IOException;


public interface IParseSource {

int consume() throws IOException;

int lookahead(int n) throws IOException;

/** @return number of characters available to consume without having to wait for a stream input,
* and with no risk of IOException if consume() is called.
*
* Use intended for tests or performance optimizations */
int bufferedCharCount();


default int lookahead() throws IOException {
return lookahead(0);
}

default boolean hasCharAhead() throws IOException {
return lookahead() != -1;
}

default char consumeNonEOF() throws IOException {
int ch = consume();
assertTrue(ch != -1);
return (char) ch;
}

default void consume(int amount) throws IOException {
while(amount-- > 0) {
consume();
}
}

default boolean lookaheadMatches(String string) throws IOException {
for(int ix = 0; ix < string.length(); ix++) {
if(lookahead(ix) != string.charAt(ix)) {
return false;
}
}
return true;
}

}
@@ -0,0 +1,77 @@
/*******************************************************************************
* Copyright (c) 2015, 2015 Bruno Medeiros and other Contributors.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Bruno Medeiros - initial API and implementation
*******************************************************************************/
package melnorme.lang.utils.parse;

import java.io.IOException;


public interface ParseSourceUtils extends IParseSource {

default boolean lookaheadIsEOF() throws IOException {
return !hasCharAhead();
}

default boolean tryConsume(String string) throws IOException {
if(lookaheadMatches(string)) {
consume(string.length());
return true;
}
return false;
}

default String consumeUntil(String string) throws IOException {
StringBuilder sb = new StringBuilder();

while(hasCharAhead() && !lookaheadMatches(string)) {
sb.append(consumeNonEOF());
}
return sb.toString();
}

default String consumeUntil(String endString, boolean consumeEndString) throws IOException {
String firstString = consumeUntil(endString);
if(consumeEndString) {
tryConsume(endString);
}
return firstString;
}

/**
* Consume a string delimited by give delimiter char,
* with given escapeChar acting a possible escape (use -1 for no escapeChar)
*/
default String consumeDelimitedString(int delimiter, int escapeChar) throws IOException {
StringBuilder sb = new StringBuilder();

while(hasCharAhead()) {

char consumedChar = consumeNonEOF();

if(consumedChar == delimiter) {
break;
}
else if(consumedChar == escapeChar && hasCharAhead()) {
int lookahead = lookahead();
if(lookahead == delimiter || lookahead == escapeChar) {

char escapedChar = consumeNonEOF();
sb.append(escapedChar);
continue;
}
}

sb.append(consumedChar);

}
return sb.toString();
}

}
Expand Up @@ -8,21 +8,21 @@
* Contributors:
* Bruno Medeiros - initial API and implementation
*******************************************************************************/
package melnorme.lang.utils;
package melnorme.lang.utils.parse;

import melnorme.lang.tests.CommonToolingTest;

import org.junit.Test;

public class SimpleLexingHelper_Test extends CommonToolingTest {
public class ParseSourceUtils_Test extends CommonToolingTest {

@Test
public void test_consumeDelimited() throws Exception { test_consumeDelimited$(); }
public void test_consumeDelimited$() throws Exception {

assertEquals(new SimpleLexingHelper("blah").consumeDelimitedString('|', '#'), "blah");
assertEquals(new StringParserHelper("blah").consumeDelimitedString('|', '#'), "blah");

SimpleLexingHelper lexingHelper = new SimpleLexingHelper("one|two|three##|four#|xxx|###|five");
StringParserHelper lexingHelper = new StringParserHelper("one|two|three##|four#|xxx|###|five");
assertEquals(lexingHelper.consumeDelimitedString('|', '#'), "one");
assertEquals(lexingHelper.consumeDelimitedString('|', '#'), "two");
assertEquals(lexingHelper.consumeDelimitedString('|', '#'), "three#");
Expand Down
@@ -0,0 +1,109 @@
/*******************************************************************************
* Copyright (c) 2015, 2015 Bruno Medeiros and other Contributors.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Bruno Medeiros - initial API and implementation
*******************************************************************************/
package melnorme.lang.utils.parse;

import static melnorme.utilbox.core.Assert.AssertNamespace.assertTrue;

import java.io.IOException;
import java.io.StringReader;

import melnorme.lang.tests.CommonToolingTest;

import org.junit.Test;

public class ParseSource_Test extends CommonToolingTest {

protected final String TEST_SOURCE = "abcdef";

protected String source;
protected int sourceIx;
protected int lookahead;


protected void doTest______(String source, IParseSource parseSource) throws IOException {

this.source = source;
this.sourceIx = 0;

// Test lookahead
checkBufferedCount(parseSource, 0);
lookahead = testLookahead(parseSource, source.charAt(sourceIx));

// Test lookahead(1)
assertTrue(parseSource.lookahead(1) == source.charAt(sourceIx + 1));

// Test consume with buffered
checkBufferedCount(parseSource, 2);
assertTrue(parseSource.consume() == lookahead); sourceIx++;
checkBufferedCount(parseSource, 1);
assertTrue(parseSource.consume() == source.charAt(sourceIx)); sourceIx++;
checkBufferedCount(parseSource, 0);


checkBufferedCount(parseSource, 0);
lookahead = testLookahead(parseSource, source.charAt(sourceIx));
// Test consume with buffered
checkBufferedCount(parseSource, 1);
assertTrue(lookahead == parseSource.consume()); sourceIx++;

assertTrue(lookahead == 'c');


checkBufferedCount(parseSource, 0);
assertTrue(parseSource.consume() == source.charAt(sourceIx));
sourceIx++;

while(sourceIx < source.length()) {
int ch = testLookahead(parseSource, source.charAt(sourceIx));
assertTrue(parseSource.consume() == ch);
sourceIx++;
}

// EOF
testLookahead(parseSource, -1);
assertTrue(parseSource.consume() == -1);
}

protected void checkBufferedCount(IParseSource parseSource, int expected) {
assertTrue(parseSource.bufferedCharCount() == expected);
}

protected int testLookahead(IParseSource parseSource, int expected) throws IOException {
int lookahead = parseSource.lookahead();
assertTrue(lookahead == expected);
assertTrue(lookahead == parseSource.lookahead(0));
return lookahead;
}

public static class ReaderParseSource_Test extends ParseSource_Test {

@Test
public void test() throws Exception { test$(); }
public void test$() throws Exception {
doTest______(TEST_SOURCE, new ReaderParseSource(new StringReader(TEST_SOURCE)));
}
}

public static class StringParseSource_Test extends ParseSource_Test {

@Test
public void test() throws Exception { test$(); }
public void test$() throws Exception {
doTest______(TEST_SOURCE, new StringParseSource(TEST_SOURCE));
}

@Override
protected void checkBufferedCount(IParseSource parseSource, int expected) {
assertTrue(parseSource.bufferedCharCount() == source.length() - sourceIx);
}
}

}

0 comments on commit e3f18ce

Please sign in to comment.