Permalink
Browse files

Simplify token test assertion helpers.

  • Loading branch information...
1 parent 0b0b1cd commit 7a4e47f41c1dc98fe281342fc0c06e7995bcc180 @plioi committed Apr 1, 2012
@@ -0,0 +1,8 @@
+namespace Parsley
+{
+ public class CharLexer : Lexer
+ {
+ public CharLexer()
+ : base(new Pattern("Character", @".")) { }
+ }
+}
@@ -1,8 +0,0 @@
-namespace Parsley
-{
- public class CharTokenStream : TokenStream
- {
- public CharTokenStream(string source)
- : base(new Lexer(new Pattern("Character", @".")).Tokenize(new Text(source))) { }
- }
-}
@@ -8,7 +8,7 @@ public class GrammarRuleTests : Grammar
[Fact]
public void CanDefineMutuallyRecursiveRules()
{
- var tokens = new CharTokenStream("(A)");
+ var tokens = new CharLexer().Tokenize(new Text("(A)"));
var expression = new GrammarRule<string>();
var alpha = new GrammarRule<string>();
var parenthesizedExpresion = new GrammarRule<string>();
@@ -1,24 +1,25 @@
using System;
+using System.Collections.Generic;
using Should;
using Xunit;
namespace Parsley
{
public class GrammarTests : Grammar
{
- private static TokenStream Tokenize(string source)
+ private static IEnumerable<Token> Tokenize(string source)
{
- return new SampleTokenStream(source);
+ return new SampleLexer().Tokenize(new Text(source));
}
- private class SampleTokenStream : TokenStream
+ private class SampleLexer : Lexer
{
public static readonly TokenKind Digit = new Pattern("Digit", @"[0-9]");
public static readonly TokenKind Letter = new Pattern("Letter", @"[a-zA-Z]");
public static readonly TokenKind Symbol = new Pattern("Symbol", @".");
- public SampleTokenStream(string source)
- : base(new Lexer(Digit, Letter, Symbol).Tokenize(new Text(source))) { }
+ public SampleLexer()
+ : base(Digit, Letter, Symbol) { }
}
private readonly Parser<Token> A, B, AB, COMMA;
@@ -51,11 +52,11 @@ public void CanDetectTheEndOfInputWithoutAdvancing()
[Fact]
public void CanDemandThatAGivenKindOfTokenAppearsNext()
{
- Token(SampleTokenStream.Letter).Parses(Tokenize("A")).IntoToken("A");
- Token(SampleTokenStream.Letter).FailsToParse(Tokenize("0")).LeavingUnparsedTokens("0").WithMessage("(1, 1): Letter expected");
+ Token(SampleLexer.Letter).Parses(Tokenize("A")).IntoToken("A");
+ Token(SampleLexer.Letter).FailsToParse(Tokenize("0")).LeavingUnparsedTokens("0").WithMessage("(1, 1): Letter expected");
- Token(SampleTokenStream.Digit).FailsToParse(Tokenize("A")).LeavingUnparsedTokens("A").WithMessage("(1, 1): Digit expected");
- Token(SampleTokenStream.Digit).Parses(Tokenize("0")).IntoToken("0");
+ Token(SampleLexer.Digit).FailsToParse(Tokenize("A")).LeavingUnparsedTokens("A").WithMessage("(1, 1): Digit expected");
+ Token(SampleLexer.Digit).Parses(Tokenize("0")).IntoToken("0");
}
[Fact]
@@ -77,7 +78,7 @@ public void ApplyingARuleZeroOrMoreTimes()
parser.FailsToParse(Tokenize("ABABA!")).LeavingUnparsedTokens("!").WithMessage("(1, 6): B expected");
Parser<Token> succeedWithoutConsuming = new LambdaParser<Token>(tokens => new Parsed<Token>(null, tokens));
- Action infiniteLoop = () => ZeroOrMore(succeedWithoutConsuming).Parse(Tokenize(""));
+ Action infiniteLoop = () => ZeroOrMore(succeedWithoutConsuming).Parse(new TokenStream(Tokenize("")));
infiniteLoop.ShouldThrow<Exception>("Parser encountered a potential infinite loop.");
}
@@ -92,7 +93,7 @@ public void ApplyingARuleOneOrMoreTimes()
parser.FailsToParse(Tokenize("ABABA!")).LeavingUnparsedTokens("!").WithMessage("(1, 6): B expected");
Parser<Token> succeedWithoutConsuming = new LambdaParser<Token>(tokens => new Parsed<Token>(null, tokens));
- Action infiniteLoop = () => OneOrMore(succeedWithoutConsuming).Parse(Tokenize(""));
+ Action infiniteLoop = () => OneOrMore(succeedWithoutConsuming).Parse(new TokenStream(Tokenize("")));
infiniteLoop.ShouldThrow<Exception>("Parser encountered a potential infinite loop.");
}
@@ -183,9 +184,9 @@ public void ImprovingDefaultMessagesWithAKnownExpectation()
public class AlternationTests : Grammar
{
- private static TokenStream Tokenize(string source)
+ private static IEnumerable<Token> Tokenize(string source)
{
- return new CharTokenStream(source);
+ return new CharLexer().Tokenize(new Text(source));
}
private readonly Parser<Token> A, B, C;
@@ -6,9 +6,9 @@ namespace Parsley.IntegrationTests.Json
{
public class JsonGrammarTests : JsonGrammar
{
- private static TokenStream Tokenize(string input)
+ private static IEnumerable<Token> Tokenize(string input)
{
- return new TokenStream(new JsonLexer().Tokenize(new Text(input)));
+ return new JsonLexer().Tokenize(new Text(input));
}
[Fact]
@@ -8,10 +8,10 @@ public class LambdaParserTests
public void CreatesParsersFromLambdas()
{
var succeeds = new LambdaParser<string>(tokens => new Parsed<string>("AA", tokens.Advance().Advance()));
- succeeds.PartiallyParses(new CharTokenStream("AABB")).LeavingUnparsedTokens("B", "B").IntoValue("AA");
+ succeeds.PartiallyParses(new CharLexer().Tokenize(new Text("AABB"))).LeavingUnparsedTokens("B", "B").IntoValue("AA");
var fails = new LambdaParser<string>(tokens => new Error<string>(tokens, ErrorMessage.Unknown()));
- fails.FailsToParse(new CharTokenStream("AABB")).LeavingUnparsedTokens("A", "A", "B", "B").WithMessage("(1, 1): Parse error.");
+ fails.FailsToParse(new CharLexer().Tokenize(new Text("AABB"))).LeavingUnparsedTokens("A", "A", "B", "B").WithMessage("(1, 1): Parse error.");
}
}
}
@@ -12,21 +12,21 @@ public OperatorPrecedenceParserTests()
{
expression = new OperatorPrecedenceParser<Expression>();
- expression.Atom(SampleTokenStream.Digit, token => new Constant(int.Parse(token.Literal)));
- expression.Atom(SampleTokenStream.Name, token => new Identifier(token.Literal));
+ expression.Atom(SampleLexer.Digit, token => new Constant(int.Parse(token.Literal)));
+ expression.Atom(SampleLexer.Name, token => new Identifier(token.Literal));
- expression.Unit(SampleTokenStream.LeftParen, Between(Token("("), expression, Token(")")));
+ expression.Unit(SampleLexer.LeftParen, Between(Token("("), expression, Token(")")));
- expression.Binary(SampleTokenStream.Add, 3, (left, symbol, right) => new Form(symbol, left, right));
- expression.Binary(SampleTokenStream.Subtract, 3, (left, symbol, right) => new Form(symbol, left, right));
- expression.Binary(SampleTokenStream.Multiply, 4, (left, symbol, right) => new Form(symbol, left, right));
- expression.Binary(SampleTokenStream.Divide, 4, (left, symbol, right) => new Form(symbol, left, right));
- expression.Binary(SampleTokenStream.Exponent, 5, (left, symbol, right) => new Form(symbol, left, right), Associativity.Right);
- expression.Prefix(SampleTokenStream.Subtract, 6, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
- expression.Postfix(SampleTokenStream.Increment, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
- expression.Postfix(SampleTokenStream.Decrement, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
+ expression.Binary(SampleLexer.Add, 3, (left, symbol, right) => new Form(symbol, left, right));
+ expression.Binary(SampleLexer.Subtract, 3, (left, symbol, right) => new Form(symbol, left, right));
+ expression.Binary(SampleLexer.Multiply, 4, (left, symbol, right) => new Form(symbol, left, right));
+ expression.Binary(SampleLexer.Divide, 4, (left, symbol, right) => new Form(symbol, left, right));
+ expression.Binary(SampleLexer.Exponent, 5, (left, symbol, right) => new Form(symbol, left, right), Associativity.Right);
+ expression.Prefix(SampleLexer.Subtract, 6, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
+ expression.Postfix(SampleLexer.Increment, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
+ expression.Postfix(SampleLexer.Decrement, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
- expression.Extend(SampleTokenStream.LeftParen, 8, callable =>
+ expression.Extend(SampleLexer.LeftParen, 8, callable =>
from arguments in Between(Token("("), ZeroOrMore(expression, Token(",")), Token(")"))
select new Form(callable, arguments));
}
@@ -121,12 +121,12 @@ private void Parses(string input, string expectedTree)
expression.Parses(Tokenize(input)).IntoValue(e => e.ToString().ShouldEqual(expectedTree));
}
- private static TokenStream Tokenize(string source)
+ private static IEnumerable<Token> Tokenize(string source)
{
- return new SampleTokenStream(source);
+ return new SampleLexer().Tokenize(new Text(source));
}
- private class SampleTokenStream : TokenStream
+ private class SampleLexer : Lexer
{
public static readonly TokenKind Digit = new Pattern("Digit", @"[0-9]");
public static readonly TokenKind Name = new Pattern("Name", @"[a-z]+");
@@ -141,10 +141,10 @@ private class SampleTokenStream : TokenStream
public static readonly TokenKind RightParen = new Operator(")");
public static readonly TokenKind Comma = new Operator(",");
- public SampleTokenStream(string source)
- : base(new Lexer(Digit, Name, Increment, Decrement, Add,
- Subtract, Multiply, Divide, Exponent,
- LeftParen, RightParen, Comma).Tokenize(new Text(source))) { }
+ public SampleLexer()
+ : base(Digit, Name, Increment, Decrement, Add,
+ Subtract, Multiply, Divide, Exponent,
+ LeftParen, RightParen, Comma) { }
}
private interface Expression
@@ -9,7 +9,7 @@ public class ParsedTests
public ParsedTests()
{
- unparsed = new CharTokenStream("0");
+ unparsed = new TokenStream(new CharLexer().Tokenize(new Text("0")));
}
[Fact]
@@ -1,4 +1,6 @@
using System;
+using System.Collections.Generic;
+using System.Linq;
using Xunit;
namespace Parsley
@@ -7,9 +9,9 @@ public class ParserQueryTests
{
private static readonly Parser<string> Next = new LambdaParser<string>(tokens => new Parsed<string>(tokens.Current.Literal, tokens.Advance()));
- private static TokenStream Tokenize(string source)
+ private static IEnumerable<Token> Tokenize(string source)
{
- return new CharTokenStream(source);
+ return new CharLexer().Tokenize(new Text(source));
}
[Fact]
@@ -45,22 +47,22 @@ public void PropogatesErrorsWithoutRunningRemainingParsers()
{
Parser<string> Fail = Grammar.Fail<string>();
- var source = Tokenize("xy");
+ var tokens = Tokenize("xy").ToArray();
(from _ in Fail
from x in Next
from y in Next
- select Tuple.Create(x, y)).FailsToParse(source).LeavingUnparsedTokens("x", "y");
+ select Tuple.Create(x, y)).FailsToParse(tokens).LeavingUnparsedTokens("x", "y");
(from x in Next
from _ in Fail
from y in Next
- select Tuple.Create(x, y)).FailsToParse(source).LeavingUnparsedTokens("y");
+ select Tuple.Create(x, y)).FailsToParse(tokens).LeavingUnparsedTokens("y");
(from x in Next
from y in Next
from _ in Fail
- select Tuple.Create(x, y)).FailsToParse(source).AtEndOfInput();
+ select Tuple.Create(x, y)).FailsToParse(tokens).AtEndOfInput();
}
}
}
@@ -47,7 +47,7 @@
<Link>Properties\CommonAssemblyInfo.cs</Link>
</Compile>
<Compile Include="AssertionExtensions.cs" />
- <Compile Include="CharTokenStream.cs" />
+ <Compile Include="CharLexer.cs" />
<Compile Include="ErrorMessageListTests.cs" />
<Compile Include="ErrorMessageTests.cs" />
<Compile Include="ErrorTests.cs" />
View
@@ -13,11 +13,6 @@ public Lexer(params TokenKind[] kinds)
this.kinds.Add(TokenKind.Unknown);
}
- public IEnumerable<Token> Tokenize(string text)
- {
- return Tokenize(new Text(text));//TODO: Should we phase out the Text overload so consumer doesn't have to care about Text ever?
- }
-
public IEnumerable<Token> Tokenize(Text text)
{
while (!text.EndOfInput)
@@ -22,9 +22,9 @@ public static void ShouldEqual(this Token actual, TokenKind expectedKind, string
AssertTokenLiteralsEqual(expectedLiteral, actual.Literal);
}
- public static Reply<T> FailsToParse<T>(this Parser<T> parser, TokenStream tokens)
+ public static Reply<T> FailsToParse<T>(this Parser<T> parser, IEnumerable<Token> tokens)
{
- var reply = parser.Parse(tokens);
+ var reply = parser.Parse(new TokenStream(tokens));
if (reply.Success)
throw new AssertionException("parser failure", "parser completed successfully");
@@ -52,14 +52,14 @@ public static Reply<T> WithNoMessage<T>(this Reply<T> reply)
return reply;
}
- public static Reply<T> PartiallyParses<T>(this Parser<T> parser, TokenStream tokens)
+ public static Reply<T> PartiallyParses<T>(this Parser<T> parser, IEnumerable<Token> tokens)
{
- return parser.Parse(tokens).Succeeds();
+ return parser.Parse(new TokenStream(tokens)).Succeeds();
}
- public static Reply<T> Parses<T>(this Parser<T> parser, TokenStream tokens)
+ public static Reply<T> Parses<T>(this Parser<T> parser, IEnumerable<Token> tokens)
{
- return parser.Parse(tokens).Succeeds().AtEndOfInput();
+ return parser.Parse(new TokenStream(tokens)).Succeeds().AtEndOfInput();
}
private static Reply<T> Succeeds<T>(this Reply<T> reply)

0 comments on commit 7a4e47f

Please sign in to comment.