From 7a4e47f41c1dc98fe281342fc0c06e7995bcc180 Mon Sep 17 00:00:00 2001 From: Patrick Lioi Date: Sat, 31 Mar 2012 20:28:47 -0500 Subject: [PATCH] Simplify token test assertion helpers. --- src/Parsley.Test/CharLexer.cs | 8 ++++ src/Parsley.Test/CharTokenStream.cs | 8 ---- src/Parsley.Test/GrammarRuleTests.cs | 2 +- src/Parsley.Test/GrammarTests.cs | 27 ++++++------- .../IntegrationTests/Json/JsonGrammarTests.cs | 4 +- src/Parsley.Test/LambdaParserTests.cs | 4 +- .../OperatorPrecedenceParserTests.cs | 38 +++++++++---------- src/Parsley.Test/ParsedTests.cs | 2 +- src/Parsley.Test/ParserQueryTests.cs | 14 ++++--- src/Parsley.Test/Parsley.Test.csproj | 2 +- src/Parsley/Lexer.cs | 5 --- src/Parsley/ParsingAssertions.cs | 12 +++--- 12 files changed, 62 insertions(+), 64 deletions(-) create mode 100644 src/Parsley.Test/CharLexer.cs delete mode 100644 src/Parsley.Test/CharTokenStream.cs diff --git a/src/Parsley.Test/CharLexer.cs b/src/Parsley.Test/CharLexer.cs new file mode 100644 index 00000000..c6e71640 --- /dev/null +++ b/src/Parsley.Test/CharLexer.cs @@ -0,0 +1,8 @@ +namespace Parsley +{ + public class CharLexer : Lexer + { + public CharLexer() + : base(new Pattern("Character", @".")) { } + } +} \ No newline at end of file diff --git a/src/Parsley.Test/CharTokenStream.cs b/src/Parsley.Test/CharTokenStream.cs deleted file mode 100644 index 1fb4b8d8..00000000 --- a/src/Parsley.Test/CharTokenStream.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Parsley -{ - public class CharTokenStream : TokenStream - { - public CharTokenStream(string source) - : base(new Lexer(new Pattern("Character", @".")).Tokenize(new Text(source))) { } - } -} \ No newline at end of file diff --git a/src/Parsley.Test/GrammarRuleTests.cs b/src/Parsley.Test/GrammarRuleTests.cs index 5f6b8aa1..09429940 100644 --- a/src/Parsley.Test/GrammarRuleTests.cs +++ b/src/Parsley.Test/GrammarRuleTests.cs @@ -8,7 +8,7 @@ public class GrammarRuleTests : Grammar [Fact] public void CanDefineMutuallyRecursiveRules() { - var tokens = new CharTokenStream("(A)"); + var tokens = new CharLexer().Tokenize(new Text("(A)")); var expression = new GrammarRule(); var alpha = new GrammarRule(); var parenthesizedExpresion = new GrammarRule(); diff --git a/src/Parsley.Test/GrammarTests.cs b/src/Parsley.Test/GrammarTests.cs index 64ff8b01..204434bc 100644 --- a/src/Parsley.Test/GrammarTests.cs +++ b/src/Parsley.Test/GrammarTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using Should; using Xunit; @@ -6,19 +7,19 @@ namespace Parsley { public class GrammarTests : Grammar { - private static TokenStream Tokenize(string source) + private static IEnumerable Tokenize(string source) { - return new SampleTokenStream(source); + return new SampleLexer().Tokenize(new Text(source)); } - private class SampleTokenStream : TokenStream + private class SampleLexer : Lexer { public static readonly TokenKind Digit = new Pattern("Digit", @"[0-9]"); public static readonly TokenKind Letter = new Pattern("Letter", @"[a-zA-Z]"); public static readonly TokenKind Symbol = new Pattern("Symbol", @"."); - public SampleTokenStream(string source) - : base(new Lexer(Digit, Letter, Symbol).Tokenize(new Text(source))) { } + public SampleLexer() + : base(Digit, Letter, Symbol) { } } private readonly Parser A, B, AB, COMMA; @@ -51,11 +52,11 @@ public void CanDetectTheEndOfInputWithoutAdvancing() [Fact] public void CanDemandThatAGivenKindOfTokenAppearsNext() { - Token(SampleTokenStream.Letter).Parses(Tokenize("A")).IntoToken("A"); - Token(SampleTokenStream.Letter).FailsToParse(Tokenize("0")).LeavingUnparsedTokens("0").WithMessage("(1, 1): Letter expected"); + Token(SampleLexer.Letter).Parses(Tokenize("A")).IntoToken("A"); + Token(SampleLexer.Letter).FailsToParse(Tokenize("0")).LeavingUnparsedTokens("0").WithMessage("(1, 1): Letter expected"); - Token(SampleTokenStream.Digit).FailsToParse(Tokenize("A")).LeavingUnparsedTokens("A").WithMessage("(1, 1): Digit expected"); - Token(SampleTokenStream.Digit).Parses(Tokenize("0")).IntoToken("0"); + Token(SampleLexer.Digit).FailsToParse(Tokenize("A")).LeavingUnparsedTokens("A").WithMessage("(1, 1): Digit expected"); + Token(SampleLexer.Digit).Parses(Tokenize("0")).IntoToken("0"); } [Fact] @@ -77,7 +78,7 @@ public void ApplyingARuleZeroOrMoreTimes() parser.FailsToParse(Tokenize("ABABA!")).LeavingUnparsedTokens("!").WithMessage("(1, 6): B expected"); Parser succeedWithoutConsuming = new LambdaParser(tokens => new Parsed(null, tokens)); - Action infiniteLoop = () => ZeroOrMore(succeedWithoutConsuming).Parse(Tokenize("")); + Action infiniteLoop = () => ZeroOrMore(succeedWithoutConsuming).Parse(new TokenStream(Tokenize(""))); infiniteLoop.ShouldThrow("Parser encountered a potential infinite loop."); } @@ -92,7 +93,7 @@ public void ApplyingARuleOneOrMoreTimes() parser.FailsToParse(Tokenize("ABABA!")).LeavingUnparsedTokens("!").WithMessage("(1, 6): B expected"); Parser succeedWithoutConsuming = new LambdaParser(tokens => new Parsed(null, tokens)); - Action infiniteLoop = () => OneOrMore(succeedWithoutConsuming).Parse(Tokenize("")); + Action infiniteLoop = () => OneOrMore(succeedWithoutConsuming).Parse(new TokenStream(Tokenize(""))); infiniteLoop.ShouldThrow("Parser encountered a potential infinite loop."); } @@ -183,9 +184,9 @@ public void ImprovingDefaultMessagesWithAKnownExpectation() public class AlternationTests : Grammar { - private static TokenStream Tokenize(string source) + private static IEnumerable Tokenize(string source) { - return new CharTokenStream(source); + return new CharLexer().Tokenize(new Text(source)); } private readonly Parser A, B, C; diff --git a/src/Parsley.Test/IntegrationTests/Json/JsonGrammarTests.cs b/src/Parsley.Test/IntegrationTests/Json/JsonGrammarTests.cs index 3d67f8fb..42456634 100644 --- a/src/Parsley.Test/IntegrationTests/Json/JsonGrammarTests.cs +++ b/src/Parsley.Test/IntegrationTests/Json/JsonGrammarTests.cs @@ -6,9 +6,9 @@ namespace Parsley.IntegrationTests.Json { public class JsonGrammarTests : JsonGrammar { - private static TokenStream Tokenize(string input) + private static IEnumerable Tokenize(string input) { - return new TokenStream(new JsonLexer().Tokenize(new Text(input))); + return new JsonLexer().Tokenize(new Text(input)); } [Fact] diff --git a/src/Parsley.Test/LambdaParserTests.cs b/src/Parsley.Test/LambdaParserTests.cs index c059affc..3ad00c64 100644 --- a/src/Parsley.Test/LambdaParserTests.cs +++ b/src/Parsley.Test/LambdaParserTests.cs @@ -8,10 +8,10 @@ public class LambdaParserTests public void CreatesParsersFromLambdas() { var succeeds = new LambdaParser(tokens => new Parsed("AA", tokens.Advance().Advance())); - succeeds.PartiallyParses(new CharTokenStream("AABB")).LeavingUnparsedTokens("B", "B").IntoValue("AA"); + succeeds.PartiallyParses(new CharLexer().Tokenize(new Text("AABB"))).LeavingUnparsedTokens("B", "B").IntoValue("AA"); var fails = new LambdaParser(tokens => new Error(tokens, ErrorMessage.Unknown())); - fails.FailsToParse(new CharTokenStream("AABB")).LeavingUnparsedTokens("A", "A", "B", "B").WithMessage("(1, 1): Parse error."); + fails.FailsToParse(new CharLexer().Tokenize(new Text("AABB"))).LeavingUnparsedTokens("A", "A", "B", "B").WithMessage("(1, 1): Parse error."); } } } \ No newline at end of file diff --git a/src/Parsley.Test/OperatorPrecedenceParserTests.cs b/src/Parsley.Test/OperatorPrecedenceParserTests.cs index ce0242d3..f32f6d0f 100644 --- a/src/Parsley.Test/OperatorPrecedenceParserTests.cs +++ b/src/Parsley.Test/OperatorPrecedenceParserTests.cs @@ -12,21 +12,21 @@ public OperatorPrecedenceParserTests() { expression = new OperatorPrecedenceParser(); - expression.Atom(SampleTokenStream.Digit, token => new Constant(int.Parse(token.Literal))); - expression.Atom(SampleTokenStream.Name, token => new Identifier(token.Literal)); + expression.Atom(SampleLexer.Digit, token => new Constant(int.Parse(token.Literal))); + expression.Atom(SampleLexer.Name, token => new Identifier(token.Literal)); - expression.Unit(SampleTokenStream.LeftParen, Between(Token("("), expression, Token(")"))); + expression.Unit(SampleLexer.LeftParen, Between(Token("("), expression, Token(")"))); - expression.Binary(SampleTokenStream.Add, 3, (left, symbol, right) => new Form(symbol, left, right)); - expression.Binary(SampleTokenStream.Subtract, 3, (left, symbol, right) => new Form(symbol, left, right)); - expression.Binary(SampleTokenStream.Multiply, 4, (left, symbol, right) => new Form(symbol, left, right)); - expression.Binary(SampleTokenStream.Divide, 4, (left, symbol, right) => new Form(symbol, left, right)); - expression.Binary(SampleTokenStream.Exponent, 5, (left, symbol, right) => new Form(symbol, left, right), Associativity.Right); - expression.Prefix(SampleTokenStream.Subtract, 6, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand)); - expression.Postfix(SampleTokenStream.Increment, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand)); - expression.Postfix(SampleTokenStream.Decrement, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand)); + expression.Binary(SampleLexer.Add, 3, (left, symbol, right) => new Form(symbol, left, right)); + expression.Binary(SampleLexer.Subtract, 3, (left, symbol, right) => new Form(symbol, left, right)); + expression.Binary(SampleLexer.Multiply, 4, (left, symbol, right) => new Form(symbol, left, right)); + expression.Binary(SampleLexer.Divide, 4, (left, symbol, right) => new Form(symbol, left, right)); + expression.Binary(SampleLexer.Exponent, 5, (left, symbol, right) => new Form(symbol, left, right), Associativity.Right); + expression.Prefix(SampleLexer.Subtract, 6, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand)); + expression.Postfix(SampleLexer.Increment, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand)); + expression.Postfix(SampleLexer.Decrement, 7, (symbol, operand) => new Form(new Identifier(symbol.Literal), operand)); - expression.Extend(SampleTokenStream.LeftParen, 8, callable => + expression.Extend(SampleLexer.LeftParen, 8, callable => from arguments in Between(Token("("), ZeroOrMore(expression, Token(",")), Token(")")) select new Form(callable, arguments)); } @@ -121,12 +121,12 @@ private void Parses(string input, string expectedTree) expression.Parses(Tokenize(input)).IntoValue(e => e.ToString().ShouldEqual(expectedTree)); } - private static TokenStream Tokenize(string source) + private static IEnumerable Tokenize(string source) { - return new SampleTokenStream(source); + return new SampleLexer().Tokenize(new Text(source)); } - private class SampleTokenStream : TokenStream + private class SampleLexer : Lexer { public static readonly TokenKind Digit = new Pattern("Digit", @"[0-9]"); public static readonly TokenKind Name = new Pattern("Name", @"[a-z]+"); @@ -141,10 +141,10 @@ private class SampleTokenStream : TokenStream public static readonly TokenKind RightParen = new Operator(")"); public static readonly TokenKind Comma = new Operator(","); - public SampleTokenStream(string source) - : base(new Lexer(Digit, Name, Increment, Decrement, Add, - Subtract, Multiply, Divide, Exponent, - LeftParen, RightParen, Comma).Tokenize(new Text(source))) { } + public SampleLexer() + : base(Digit, Name, Increment, Decrement, Add, + Subtract, Multiply, Divide, Exponent, + LeftParen, RightParen, Comma) { } } private interface Expression diff --git a/src/Parsley.Test/ParsedTests.cs b/src/Parsley.Test/ParsedTests.cs index ead7ba62..51fc7ec8 100644 --- a/src/Parsley.Test/ParsedTests.cs +++ b/src/Parsley.Test/ParsedTests.cs @@ -9,7 +9,7 @@ public class ParsedTests public ParsedTests() { - unparsed = new CharTokenStream("0"); + unparsed = new TokenStream(new CharLexer().Tokenize(new Text("0"))); } [Fact] diff --git a/src/Parsley.Test/ParserQueryTests.cs b/src/Parsley.Test/ParserQueryTests.cs index 5e9044c8..2570f12e 100644 --- a/src/Parsley.Test/ParserQueryTests.cs +++ b/src/Parsley.Test/ParserQueryTests.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; using Xunit; namespace Parsley @@ -7,9 +9,9 @@ public class ParserQueryTests { private static readonly Parser Next = new LambdaParser(tokens => new Parsed(tokens.Current.Literal, tokens.Advance())); - private static TokenStream Tokenize(string source) + private static IEnumerable Tokenize(string source) { - return new CharTokenStream(source); + return new CharLexer().Tokenize(new Text(source)); } [Fact] @@ -45,22 +47,22 @@ public void PropogatesErrorsWithoutRunningRemainingParsers() { Parser Fail = Grammar.Fail(); - var source = Tokenize("xy"); + var tokens = Tokenize("xy").ToArray(); (from _ in Fail from x in Next from y in Next - select Tuple.Create(x, y)).FailsToParse(source).LeavingUnparsedTokens("x", "y"); + select Tuple.Create(x, y)).FailsToParse(tokens).LeavingUnparsedTokens("x", "y"); (from x in Next from _ in Fail from y in Next - select Tuple.Create(x, y)).FailsToParse(source).LeavingUnparsedTokens("y"); + select Tuple.Create(x, y)).FailsToParse(tokens).LeavingUnparsedTokens("y"); (from x in Next from y in Next from _ in Fail - select Tuple.Create(x, y)).FailsToParse(source).AtEndOfInput(); + select Tuple.Create(x, y)).FailsToParse(tokens).AtEndOfInput(); } } } \ No newline at end of file diff --git a/src/Parsley.Test/Parsley.Test.csproj b/src/Parsley.Test/Parsley.Test.csproj index e2eee550..2a8ad25b 100644 --- a/src/Parsley.Test/Parsley.Test.csproj +++ b/src/Parsley.Test/Parsley.Test.csproj @@ -47,7 +47,7 @@ Properties\CommonAssemblyInfo.cs - + diff --git a/src/Parsley/Lexer.cs b/src/Parsley/Lexer.cs index 8a6a48fc..bed4ce16 100644 --- a/src/Parsley/Lexer.cs +++ b/src/Parsley/Lexer.cs @@ -13,11 +13,6 @@ public Lexer(params TokenKind[] kinds) this.kinds.Add(TokenKind.Unknown); } - public IEnumerable Tokenize(string text) - { - return Tokenize(new Text(text));//TODO: Should we phase out the Text overload so consumer doesn't have to care about Text ever? - } - public IEnumerable Tokenize(Text text) { while (!text.EndOfInput) diff --git a/src/Parsley/ParsingAssertions.cs b/src/Parsley/ParsingAssertions.cs index 2043820d..ad1a3c8e 100644 --- a/src/Parsley/ParsingAssertions.cs +++ b/src/Parsley/ParsingAssertions.cs @@ -22,9 +22,9 @@ public static void ShouldEqual(this Token actual, TokenKind expectedKind, string AssertTokenLiteralsEqual(expectedLiteral, actual.Literal); } - public static Reply FailsToParse(this Parser parser, TokenStream tokens) + public static Reply FailsToParse(this Parser parser, IEnumerable tokens) { - var reply = parser.Parse(tokens); + var reply = parser.Parse(new TokenStream(tokens)); if (reply.Success) throw new AssertionException("parser failure", "parser completed successfully"); @@ -52,14 +52,14 @@ public static Reply WithNoMessage(this Reply reply) return reply; } - public static Reply PartiallyParses(this Parser parser, TokenStream tokens) + public static Reply PartiallyParses(this Parser parser, IEnumerable tokens) { - return parser.Parse(tokens).Succeeds(); + return parser.Parse(new TokenStream(tokens)).Succeeds(); } - public static Reply Parses(this Parser parser, TokenStream tokens) + public static Reply Parses(this Parser parser, IEnumerable tokens) { - return parser.Parse(tokens).Succeeds().AtEndOfInput(); + return parser.Parse(new TokenStream(tokens)).Succeeds().AtEndOfInput(); } private static Reply Succeeds(this Reply reply)