Simplify token test assertion helpers.

src/Parsley.Test/CharLexer.cs

@@ -0,0 +1,8 @@
+namespace Parsley
+{
+    public class CharLexer : Lexer
+    {
+        public CharLexer()
+            : base(new Pattern("Character", @".")) { }
+    }
+}

src/Parsley.Test/GrammarRuleTests.cs

@@ -8,7 +8,7 @@ public class GrammarRuleTests : Grammar
         [Fact]
         public void CanDefineMutuallyRecursiveRules()
         {
-            var tokens = new CharTokenStream("(A)");
+            var tokens = new CharLexer().Tokenize(new Text("(A)"));
             var expression = new GrammarRule<string>();
             var alpha = new GrammarRule<string>();
             var parenthesizedExpresion = new GrammarRule<string>();

src/Parsley.Test/GrammarTests.cs

@@ -1,24 +1,25 @@
 using System;
+using System.Collections.Generic;
 using Should;
 using Xunit;
 
 namespace Parsley
 {
     public class GrammarTests : Grammar
     {
-        private static TokenStream Tokenize(string source)
+        private static IEnumerable<Token> Tokenize(string source)
         {
-            return new SampleTokenStream(source);
+            return new SampleLexer().Tokenize(new Text(source));
         }
 
-        private class SampleTokenStream : TokenStream
+        private class SampleLexer : Lexer
         {
             public static readonly TokenKind Digit = new Pattern("Digit", @"[0-9]");
             public static readonly TokenKind Letter = new Pattern("Letter", @"[a-zA-Z]");
             public static readonly TokenKind Symbol = new Pattern("Symbol", @".");
 
-            public SampleTokenStream(string source)
-                : base(new Lexer(Digit, Letter, Symbol).Tokenize(new Text(source))) { }
+            public SampleLexer()
+                : base(Digit, Letter, Symbol) { }
         }
 
         private readonly Parser<Token> A, B, AB, COMMA;
@@ -51,11 +52,11 @@ public void CanDetectTheEndOfInputWithoutAdvancing()
         [Fact]
         public void CanDemandThatAGivenKindOfTokenAppearsNext()
         {
-            Token(SampleTokenStream.Letter).Parses(Tokenize("A")).IntoToken("A");
-            Token(SampleTokenStream.Letter).FailsToParse(Tokenize("0")).LeavingUnparsedTokens("0").WithMessage("(1, 1): Letter expected");
+            Token(SampleLexer.Letter).Parses(Tokenize("A")).IntoToken("A");
+            Token(SampleLexer.Letter).FailsToParse(Tokenize("0")).LeavingUnparsedTokens("0").WithMessage("(1, 1): Letter expected");
 
-            Token(SampleTokenStream.Digit).FailsToParse(Tokenize("A")).LeavingUnparsedTokens("A").WithMessage("(1, 1): Digit expected");
-            Token(SampleTokenStream.Digit).Parses(Tokenize("0")).IntoToken("0");
+            Token(SampleLexer.Digit).FailsToParse(Tokenize("A")).LeavingUnparsedTokens("A").WithMessage("(1, 1): Digit expected");
+            Token(SampleLexer.Digit).Parses(Tokenize("0")).IntoToken("0");
         }
 
         [Fact]
@@ -77,7 +78,7 @@ public void ApplyingARuleZeroOrMoreTimes()
             parser.FailsToParse(Tokenize("ABABA!")).LeavingUnparsedTokens("!").WithMessage("(1, 6): B expected");
 
             Parser<Token> succeedWithoutConsuming = new LambdaParser<Token>(tokens => new Parsed<Token>(null, tokens));
-            Action infiniteLoop = () => ZeroOrMore(succeedWithoutConsuming).Parse(Tokenize(""));
+            Action infiniteLoop = () => ZeroOrMore(succeedWithoutConsuming).Parse(new TokenStream(Tokenize("")));
             infiniteLoop.ShouldThrow<Exception>("Parser encountered a potential infinite loop.");
         }
 
@@ -92,7 +93,7 @@ public void ApplyingARuleOneOrMoreTimes()
             parser.FailsToParse(Tokenize("ABABA!")).LeavingUnparsedTokens("!").WithMessage("(1, 6): B expected");
 
             Parser<Token> succeedWithoutConsuming = new LambdaParser<Token>(tokens => new Parsed<Token>(null, tokens));
-            Action infiniteLoop = () => OneOrMore(succeedWithoutConsuming).Parse(Tokenize(""));
+            Action infiniteLoop = () => OneOrMore(succeedWithoutConsuming).Parse(new TokenStream(Tokenize("")));
             infiniteLoop.ShouldThrow<Exception>("Parser encountered a potential infinite loop.");
         }
 
@@ -183,9 +184,9 @@ public void ImprovingDefaultMessagesWithAKnownExpectation()
 
     public class AlternationTests : Grammar
     {
-        private static TokenStream Tokenize(string source)
+        private static IEnumerable<Token> Tokenize(string source)
         {
-            return new CharTokenStream(source);
+            return new CharLexer().Tokenize(new Text(source));
         }
 
         private readonly Parser<Token> A, B, C;

src/Parsley.Test/IntegrationTests/Json/JsonGrammarTests.cs

@@ -6,9 +6,9 @@ namespace Parsley.IntegrationTests.Json
 {
     public class JsonGrammarTests : JsonGrammar
     {
-        private static TokenStream Tokenize(string input)
+        private static IEnumerable<Token> Tokenize(string input)
         {
-            return new TokenStream(new JsonLexer().Tokenize(new Text(input)));
+            return new JsonLexer().Tokenize(new Text(input));
         }
 
         [Fact]

src/Parsley.Test/LambdaParserTests.cs

@@ -8,10 +8,10 @@ public class LambdaParserTests
         public void CreatesParsersFromLambdas()
         {
             var succeeds = new LambdaParser<string>(tokens => new Parsed<string>("AA", tokens.Advance().Advance()));
-            succeeds.PartiallyParses(new CharTokenStream("AABB")).LeavingUnparsedTokens("B", "B").IntoValue("AA");
+            succeeds.PartiallyParses(new CharLexer().Tokenize(new Text("AABB"))).LeavingUnparsedTokens("B", "B").IntoValue("AA");
 
             var fails = new LambdaParser<string>(tokens => new Error<string>(tokens, ErrorMessage.Unknown()));
-            fails.FailsToParse(new CharTokenStream("AABB")).LeavingUnparsedTokens("A", "A", "B", "B").WithMessage("(1, 1): Parse error.");
+            fails.FailsToParse(new CharLexer().Tokenize(new Text("AABB"))).LeavingUnparsedTokens("A", "A", "B", "B").WithMessage("(1, 1): Parse error.");
         }
     }
 }

src/Parsley.Test/OperatorPrecedenceParserTests.cs

@@ -12,21 +12,21 @@ public OperatorPrecedenceParserTests()
         {
             expression = new OperatorPrecedenceParser<Expression>();
 
-            expression.Atom(SampleTokenStream.Digit, token => new Constant(int.Parse(token.Literal)));
-            expression.Atom(SampleTokenStream.Name, token => new Identifier(token.Literal));
+            expression.Atom(SampleLexer.Digit, token => new Constant(int.Parse(token.Literal)));
+            expression.Atom(SampleLexer.Name, token => new Identifier(token.Literal));
 
-            expression.Unit(SampleTokenStream.LeftParen, Between(Token("("), expression, Token(")")));
+            expression.Unit(SampleLexer.LeftParen, Between(Token("("), expression, Token(")")));
 
-            expression.Binary(SampleTokenStream.Add, 3, (left, symbol, right) => new Form(symbol, left, right));
-            expression.Binary(SampleTokenStream.Subtract, 3, (left, symbol, right) => new Form(symbol, left, right));
-            expression.Binary(SampleTokenStream.Multiply, 4, (left, symbol, right) => new Form(symbol, left, right));
-            expression.Binary(SampleTokenStream.Divide, 4, (left, symbol, right) => new Form(symbol, left, right));
-            expression.Binary(SampleTokenStream.Exponent, 5, (left, symbol, right) => new Form(symbol, left, right), Associativity.Right);
-            expression.Prefix(SampleTokenStream.Subtract, 6, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
-            expression.Postfix(SampleTokenStream.Increment, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
-            expression.Postfix(SampleTokenStream.Decrement, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
+            expression.Binary(SampleLexer.Add, 3, (left, symbol, right) => new Form(symbol, left, right));
+            expression.Binary(SampleLexer.Subtract, 3, (left, symbol, right) => new Form(symbol, left, right));
+            expression.Binary(SampleLexer.Multiply, 4, (left, symbol, right) => new Form(symbol, left, right));
+            expression.Binary(SampleLexer.Divide, 4, (left, symbol, right) => new Form(symbol, left, right));
+            expression.Binary(SampleLexer.Exponent, 5, (left, symbol, right) => new Form(symbol, left, right), Associativity.Right);
+            expression.Prefix(SampleLexer.Subtract, 6, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
+            expression.Postfix(SampleLexer.Increment, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
+            expression.Postfix(SampleLexer.Decrement, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand));
 
-            expression.Extend(SampleTokenStream.LeftParen, 8, callable =>
+            expression.Extend(SampleLexer.LeftParen, 8, callable =>
                                  from arguments in Between(Token("("), ZeroOrMore(expression, Token(",")), Token(")"))
                                  select new Form(callable, arguments));
         }
@@ -121,12 +121,12 @@ private void Parses(string input, string expectedTree)
             expression.Parses(Tokenize(input)).IntoValue(e => e.ToString().ShouldEqual(expectedTree));
         }
 
-        private static TokenStream Tokenize(string source)
+        private static IEnumerable<Token> Tokenize(string source)
         {
-            return new SampleTokenStream(source);
+            return new SampleLexer().Tokenize(new Text(source));
         }
 
-        private class SampleTokenStream : TokenStream
+        private class SampleLexer : Lexer
         {
             public static readonly TokenKind Digit = new Pattern("Digit", @"[0-9]");
             public static readonly TokenKind Name = new Pattern("Name", @"[a-z]+");
@@ -141,10 +141,10 @@ private class SampleTokenStream : TokenStream
             public static readonly TokenKind RightParen = new Operator(")");
             public static readonly TokenKind Comma = new Operator(",");
 
-            public SampleTokenStream(string source)
-                : base(new Lexer(Digit, Name, Increment, Decrement, Add,
-                                 Subtract, Multiply, Divide, Exponent,
-                                 LeftParen, RightParen, Comma).Tokenize(new Text(source))) { }
+            public SampleLexer()
+                : base(Digit, Name, Increment, Decrement, Add,
+                       Subtract, Multiply, Divide, Exponent,
+                       LeftParen, RightParen, Comma) { }
         }
 
         private interface Expression

src/Parsley.Test/ParsedTests.cs

@@ -9,7 +9,7 @@ public class ParsedTests
 
         public ParsedTests()
         {
-            unparsed = new CharTokenStream("0");
+            unparsed = new TokenStream(new CharLexer().Tokenize(new Text("0")));
         }
 
         [Fact]

src/Parsley.Test/ParserQueryTests.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using Xunit;
 
 namespace Parsley
@@ -7,9 +9,9 @@ public class ParserQueryTests
     {
         private static readonly Parser<string> Next = new LambdaParser<string>(tokens => new Parsed<string>(tokens.Current.Literal, tokens.Advance()));
 
-        private static TokenStream Tokenize(string source)
+        private static IEnumerable<Token> Tokenize(string source)
         {
-            return new CharTokenStream(source);
+            return new CharLexer().Tokenize(new Text(source));
         }
 
         [Fact]
@@ -45,22 +47,22 @@ public void PropogatesErrorsWithoutRunningRemainingParsers()
         {
             Parser<string> Fail = Grammar.Fail<string>();
 
-            var source = Tokenize("xy");
+            var tokens = Tokenize("xy").ToArray();
 
             (from _ in Fail
              from x in Next
              from y in Next
-             select Tuple.Create(x, y)).FailsToParse(source).LeavingUnparsedTokens("x", "y");
+             select Tuple.Create(x, y)).FailsToParse(tokens).LeavingUnparsedTokens("x", "y");
 
             (from x in Next
              from _ in Fail
              from y in Next
-             select Tuple.Create(x, y)).FailsToParse(source).LeavingUnparsedTokens("y");
+             select Tuple.Create(x, y)).FailsToParse(tokens).LeavingUnparsedTokens("y");
 
             (from x in Next
              from y in Next
              from _ in Fail
-             select Tuple.Create(x, y)).FailsToParse(source).AtEndOfInput();
+             select Tuple.Create(x, y)).FailsToParse(tokens).AtEndOfInput();
         }
     }
 }

src/Parsley.Test/Parsley.Test.csproj

@@ -47,7 +47,7 @@
       <Link>Properties\CommonAssemblyInfo.cs</Link>
     </Compile>
     <Compile Include="AssertionExtensions.cs" />
-    <Compile Include="CharTokenStream.cs" />
+    <Compile Include="CharLexer.cs" />
     <Compile Include="ErrorMessageListTests.cs" />
     <Compile Include="ErrorMessageTests.cs" />
     <Compile Include="ErrorTests.cs" />

src/Parsley/Lexer.cs

@@ -13,11 +13,6 @@ public Lexer(params TokenKind[] kinds)
             this.kinds.Add(TokenKind.Unknown);
         }
 
-        public IEnumerable<Token> Tokenize(string text)
-        {
-            return Tokenize(new Text(text));//TODO: Should we phase out the Text overload so consumer doesn't have to care about Text ever?
-        }
-
         public IEnumerable<Token> Tokenize(Text text)
         {
             while (!text.EndOfInput)

src/Parsley/ParsingAssertions.cs

@@ -22,9 +22,9 @@ public static void ShouldEqual(this Token actual, TokenKind expectedKind, string
             AssertTokenLiteralsEqual(expectedLiteral, actual.Literal);
         }
 
-        public static Reply<T> FailsToParse<T>(this Parser<T> parser, TokenStream tokens)
+        public static Reply<T> FailsToParse<T>(this Parser<T> parser, IEnumerable<Token> tokens)
         {
-            var reply = parser.Parse(tokens);
+            var reply = parser.Parse(new TokenStream(tokens));
 
             if (reply.Success)
                 throw new AssertionException("parser failure", "parser completed successfully");
@@ -52,14 +52,14 @@ public static Reply<T> WithNoMessage<T>(this Reply<T> reply)
             return reply;
         }
 
-        public static Reply<T> PartiallyParses<T>(this Parser<T> parser, TokenStream tokens)
+        public static Reply<T> PartiallyParses<T>(this Parser<T> parser, IEnumerable<Token> tokens)
         {
-            return parser.Parse(tokens).Succeeds();
+            return parser.Parse(new TokenStream(tokens)).Succeeds();
         }
 
-        public static Reply<T> Parses<T>(this Parser<T> parser, TokenStream tokens)
+        public static Reply<T> Parses<T>(this Parser<T> parser, IEnumerable<Token> tokens)
         {
-            return parser.Parse(tokens).Succeeds().AtEndOfInput();
+            return parser.Parse(new TokenStream(tokens)).Succeeds().AtEndOfInput();
         }
 
         private static Reply<T> Succeeds<T>(this Reply<T> reply)