Skip to content

Commit

Permalink
* Implemented test cases (group grammar) for whitespace handling (see
Browse files Browse the repository at this point in the history 
#23)
* Made EOF parser token invisible in parse tree (fixes #26)
* Better naming of instance variables (Supplier<Parser> is a
'parserSupplier', not a 'parser')
  • Loading branch information
@danieldietrich
danieldietrich committed Sep 14, 2014
1 parent 4e5d504 commit b6f7a04
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 22 deletions.
43 changes: 24 additions & 19 deletions src/main/java/javaslang/parser/Parser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@
*/
package javaslang.parser;

import static java.util.Collections.emptyList;
import static java.util.stream.Collectors.joining;
import static javaslang.Requirements.require;
import static javaslang.Requirements.requireNonNull;
import static javaslang.Requirements.requireNotNullOrEmpty;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Predicate;
import java.util.function.Supplier;
Expand Down Expand Up @@ -236,7 +236,7 @@ private EOF() {
@Override
public Either<Integer, List<Node<Token>>> parse(String text, int index, boolean lex) {
final boolean match = (index == text.length());
return match ? symbol(EOF, text, index, 0) : stoppedAt(index);
return match ? token() : stoppedAt(index);
}

@Override
Expand Down Expand Up @@ -282,27 +282,27 @@ public String toString() {
*/
static class Quantifier implements HasChildren, Parser {

final Supplier<Parser> parser;
final Supplier<Parser> parserSupplier;
final Bounds bounds;

Quantifier(Supplier<Parser> parser, Bounds bounds) {
requireNonNull(parser, "parser is null");
Quantifier(Supplier<Parser> parserSupplier, Bounds bounds) {
requireNonNull(parserSupplier, "parserSupplier is null");
requireNonNull(bounds, "bounds is null");
this.parser = parser;
this.parserSupplier = parserSupplier;
this.bounds = bounds;
}

@Override
public List<Supplier<Parser>> getChildren() {
return Arrays.asList(parser);
return Arrays.asList(parserSupplier);
}

@Override
public Either<Integer, List<Node<Token>>> parse(String text, int index, boolean lex) {
final Either<Integer, List<Node<Token>>> parsed = read(parser.get(), text, index, lex);
final Either<Integer, List<Node<Token>>> parsed = read(parserSupplier.get(), text, index, lex);
if (parsed.isLeft()) {
// no token found: 1..n => not ok, 0..1, 0..n => ok (with length 0)
return bounds.required ? stoppedAt(index) : new Right<>(emptyList());
return bounds.required ? stoppedAt(index) : token();
} else {
return (lex ? Transformer.COMBINER : Transformer.AGGREGATOR).apply(text, index, parsed);
}
Expand All @@ -328,7 +328,7 @@ private Either<Integer, List<Node<Token>>> read(Parser parser, String text, int

@Override
public String toString() {
return toString(parser.get()) + bounds.symbol;
return toString(parserSupplier.get()) + bounds.symbol;
}

private String toString(Parser parser) {
Expand Down Expand Up @@ -489,18 +489,18 @@ static class Sequence implements HasChildren, Parser {
static final Parser DEFAULT_WS = new Rule("WS", new Quantifier(new Charset(" \t\r\n"),
Quantifier.Bounds.ZERO_TO_N));

final Supplier<Parser>[] parsers;
final Supplier<Parser>[] parserSuppliers;

@SafeVarargs
public Sequence(Supplier<Parser>... parsers) {
requireNonNull(parsers, "no parsers");
require(parsers.length >= 2, "number of parsers < 2");
this.parsers = parsers;
public Sequence(Supplier<Parser>... parserSuppliers) {
requireNonNull(parserSuppliers, "parserSuppliers is null");
require(parserSuppliers.length >= 2, "number of parserSuppliers < 2");
this.parserSuppliers = parserSuppliers;
}

@Override
public List<Supplier<Parser>> getChildren() {
return Arrays.asList(parsers);
return Arrays.asList(parserSuppliers);
}

@Override
Expand All @@ -516,12 +516,12 @@ public Either<Integer, List<Node<Token>>> parse(String text, int index, boolean
private Either<Integer, List<Node<Token>>> read(String text, int index, boolean lex) {
final List<Node<Token>> tokens = new ArrayList<>();
int currentIndex = index;
for (Supplier<Parser> parser : parsers) {
for (Supplier<Parser> parserSupplier : parserSuppliers) {
// TODO: issue #23: fix whitespace handling
if (!lex) {
currentIndex = skipWhitespace(text, currentIndex);
}
final Either<Integer, List<Node<Token>>> parsed = parser.get().parse(text, currentIndex, lex);
final Either<Integer, List<Node<Token>>> parsed = parserSupplier.get().parse(text, currentIndex, lex);
if (parsed.isRight()) {
tokens.addAll(parsed.get());
currentIndex = endIndex(tokens).orElse(currentIndex);
Expand All @@ -538,7 +538,7 @@ private int skipWhitespace(String text, int index) {

@Override
public String toString() {
return Stream.of(parsers).map(supplier -> {
return Stream.of(parserSuppliers).map(supplier -> {
final Parser parser = supplier.get();
if (parser instanceof Rule) {
return ((Rule) parser).name;
Expand Down Expand Up @@ -601,6 +601,11 @@ static interface HasChildren {

// -- parse-result factory methods

// represents empty token having no nodes / no text
static Either<Integer, List<Node<Token>>> token() {
return new Right<>(Collections.emptyList());
}

// terminal token / leaf of the parse tree
static Either<Integer, List<Node<Token>>> token(String text, int index, int length) {
return new Right<>(Arrays.asList(new Node<>(new Token(null, text, index, length))));
Expand Down
55 changes: 53 additions & 2 deletions src/test/java/javaslang/parser/GrammarTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import javaslang.monad.Either;
import javaslang.monad.Try;

import org.junit.Ignore;
import org.junit.Test;

public class GrammarTest {
Expand All @@ -31,8 +32,8 @@ public void shouldStringifyGrammar() {

@Test
public void shouldParseTextWhenMatching() {
assertThat(new Grammar(Grammar.rule("root", Grammar.EOF)).parse("").toString()).isEqualTo(
"Success(Tree(root EOF))");
assertThat(new Grammar(Grammar.rule("root", Grammar.EOF)).parse("").toString())
.isEqualTo("Success(Tree(root))");
}

@Test
Expand Down Expand Up @@ -134,6 +135,22 @@ public void shouldCreateDelimitedListWithPrefixAndSuffix() {
assertThat(Grammar.list(Grammar.ANY, ",", "{", "}").toString()).isEqualTo("'{' ( . ( ',' . )* )? '}'");
}

// -- whitespace

@Test
@Ignore
public void shouldParseGroupsWithoutWhitespace() {
final String actual = new GroupGrammar().parse("(abc)(def ghi)").get().toString();
/* TODO:DEBUG */System.out.println(actual);
}

@Test
@Ignore
public void shouldParseGroupsWithWhitespace() {
final String actual = new GroupGrammar().parse("( abc ) ( def ghi )").get().toString();
/* TODO:DEBUG */System.out.println(actual);
}

// -- Example grammar: Simple sequence of tokens

static class SimpleSequenceGrammar extends Grammar {
Expand Down Expand Up @@ -201,8 +218,42 @@ static Parser NAME() {
}
}

/**
* Grammar for groups of words.
*
* <pre>
* <code>
* groups : group* EOF
*
* group : '(' WORD+ ')'
*
* WORD : 'a'..'z'+
* </code>
* </pre>
*/
// TODO: test issue #23 "[parser] Fix whitespace handling" - https://github.com/rocketscience-projects/javaslang/issues/23
static class GroupGrammar extends Grammar {
// define start rule
GroupGrammar() {
super(GroupGrammar::groups);
}

static Parser.Rule groups() {
return rule("groups", seq(_0_n(GroupGrammar::group), EOF));
}

static Parser.Rule group() {
return rule("group", seq(str("("), _1_n(GroupGrammar::WORD), str(")")));
}

static Parser.Rule WORD() {
return rule("WORD", _1_n(range('a', 'z')));
}
}

// -- Example grammar: Resursive expressions

// TODO: test issue #32 "[parser] Support direct and indirect recursion" - https://github.com/rocketscience-projects/javaslang/issues/32
static class ExpressionGrammar extends Grammar {

// define start rule
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/javaslang/parser/ParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ public void shouldParseFirstAlternativeUsingSubrule() {
@Test
public void shouldParseSecondAlternativeUsingSubrule() {
final Parser.Subrule subrule = new Parser.Subrule(Parser.Any.INSTANCE, Parser.EOF.INSTANCE);
assertThat(subrule.parse("", 0, false).toString()).isEqualTo("Right([Node(EOF)])");
assertThat(subrule.parse("", 0, false).toString()).isEqualTo("Right([])");
}

@Test
Expand Down

0 comments on commit b6f7a04

Please sign in to comment.