diff --git a/ParserTests/comments/CommentsTestAlternative.cs b/ParserTests/comments/CommentsTestAlternative.cs index 708d811d..d195c89d 100644 --- a/ParserTests/comments/CommentsTestAlternative.cs +++ b/ParserTests/comments/CommentsTestAlternative.cs @@ -34,7 +34,9 @@ public void NotEndingMultiComment() 2 /* not ending comment"; - var tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(4, tokens.Count); @@ -73,7 +75,9 @@ public void TestGenericMultiLineComment() 2 /* multi line comment on 2 lines */ 3.0"; - var tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(5, tokens.Count); @@ -111,9 +115,11 @@ public void TestGenericSingleLineComment() var dump = lexer.ToString(); - var tokens = lexer.Tokenize(@"1 + var r = lexer.Tokenize(@"1 2 // single line comment -3.0").ToList(); +3.0"); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(5, tokens.Count); @@ -154,7 +160,9 @@ public void TestInnerMultiComment() 4 "; - var tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(6, tokens.Count); @@ -199,8 +207,9 @@ public void TestMixedEOLComment() var dump = lexer.ToString(); var code = "1\n2\r\n/* multi line \rcomment on 2 lines */ 3.0"; - List> tokens = null; - tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(5, tokens.Count); diff --git a/ParserTests/comments/CommentsTestGeneric.cs b/ParserTests/comments/CommentsTestGeneric.cs index bfd4d2ba..f9f258c0 100644 --- a/ParserTests/comments/CommentsTestGeneric.cs +++ b/ParserTests/comments/CommentsTestGeneric.cs @@ -33,7 +33,9 @@ public void NotEndingMultiComment() 2 /* not ending comment"; - var tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(4, tokens.Count); @@ -72,7 +74,9 @@ public void TestGenericMultiLineComment() 2 /* multi line comment on 2 lines */ 3.0"; - var tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(5, tokens.Count); @@ -111,9 +115,11 @@ public void TestGenericSingleLineComment() var dump = lexer.ToString(); - var tokens = lexer.Tokenize(@"1 + var r = lexer.Tokenize(@"1 2 // single line comment -3.0").ToList(); +3.0"); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(5, tokens.Count); @@ -156,7 +162,9 @@ public void TestInnerMultiComment() 4 "; - var tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(6, tokens.Count); @@ -203,8 +211,10 @@ public void TestMixedEOLComment() var dump = lexer.ToString(); var code = "1\n2\r\n/* multi line \rcomment on 2 lines */ 3.0"; - List> tokens = null; - tokens = lexer.Tokenize(code).ToList(); + + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(5, tokens.Count); diff --git a/ParserTests/comments/MultiLineCommentsTest.cs b/ParserTests/comments/MultiLineCommentsTest.cs index 33303d96..33264d1d 100644 --- a/ParserTests/comments/MultiLineCommentsTest.cs +++ b/ParserTests/comments/MultiLineCommentsTest.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using sly.buildresult; @@ -33,7 +34,9 @@ public void NotEndingMultiComment() 2 /* not ending comment"; - var tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(4, tokens.Count); @@ -72,7 +75,9 @@ public void TestGenericMultiLineComment() 2 /* multi line comment on 2 lines */ 3.0"; - var tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(5, tokens.Count); @@ -110,13 +115,11 @@ public void TestGenericSingleLineComment() var dump = lexer.ToString(); - var error = Assert.Throws(() => - { - lexer.Tokenize(@"1 + var r = lexer.Tokenize(@"1 2 // single line comment 3.0"); - }); - Assert.Equal('/', error.Error.UnexpectedChar); + Assert.True(r.IsError); + Assert.Equal('/', r.Error.UnexpectedChar); } [Fact] @@ -133,8 +136,9 @@ public void TestInnerMultiComment() 4 "; - var tokens = lexer.Tokenize(code).ToList(); - + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(6, tokens.Count); var token1 = tokens[0]; @@ -178,8 +182,9 @@ public void TestMixedEOLComment() var dump = lexer.ToString(); var code = "1\n2\r\n/* multi line \rcomment on 2 lines */ 3.0"; - List> tokens = null; - tokens = lexer.Tokenize(code).ToList(); + var r = lexer.Tokenize(code); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(5, tokens.Count); diff --git a/ParserTests/comments/SingleLineCommentsTest.cs b/ParserTests/comments/SingleLineCommentsTest.cs index 1055fbc9..a740dbfe 100644 --- a/ParserTests/comments/SingleLineCommentsTest.cs +++ b/ParserTests/comments/SingleLineCommentsTest.cs @@ -30,13 +30,12 @@ public void TestGenericMultiLineCommentWithSingleLineComment() var dump = lexer.ToString(); - var error = Assert.Throws(() => - { - lexer?.Tokenize(@"1 + var r =lexer?.Tokenize(@"1 2 /* multi line -comment on 2 lines */ 3.0").ToList(); - }); - Assert.Equal('*', error.Error.UnexpectedChar); +comment on 2 lines */ 3.0"); + Assert.True(r.IsError); + var tokens = r.Tokens; + Assert.Equal('*', r.Error.UnexpectedChar); } [Fact] @@ -48,9 +47,11 @@ public void TestGenericSingleLineComment() var dump = lexer.ToString(); - var tokens = lexer.Tokenize(@"1 + var r = lexer.Tokenize(@"1 2 // single line comment -3.0").ToList(); +3.0"); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(5, tokens.Count); diff --git a/ParserTests/lexer/GenericLexerTests.cs b/ParserTests/lexer/GenericLexerTests.cs index c483a12d..da479c02 100644 --- a/ParserTests/lexer/GenericLexerTests.cs +++ b/ParserTests/lexer/GenericLexerTests.cs @@ -96,6 +96,11 @@ public static void AddExtension(Extensions token, LexemeAttribute lexem, Generic } } + + public enum StringDelimiters { + [Lexeme(GenericToken.String,"'","'")] + MyString + } public enum BadLetterStringDelimiter { [Lexeme(GenericToken.String, "a")] Letter @@ -188,9 +193,11 @@ public void TestAlphaId() var lexerRes = LexerBuilder.BuildLexer(new BuildResult>()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; - var r = lexer.Tokenize("alpha").ToList(); - Assert.Equal(2, r.Count); - var tok = r[0]; + + var r = lexer.Tokenize("alpha"); + Assert.True(r.IsOk); + Assert.Equal(2, r.Tokens.Count); + var tok = r.Tokens[0]; Assert.Equal(AlphaId.ID, tok.TokenID); Assert.Equal("alpha", tok.StringWithoutQuotes); ; @@ -202,9 +209,10 @@ public void TestAlphaNumDashId() var lexerRes = LexerBuilder.BuildLexer(new BuildResult>()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; - var r = lexer.Tokenize("alpha-123_").ToList(); - Assert.Equal(2, r.Count); - var tok = r[0]; + var r = lexer.Tokenize("alpha-123_"); + Assert.True(r.IsOk); + Assert.Equal(2, r.Tokens.Count); + var tok = r.Tokens[0]; Assert.Equal(AlphaNumDashId.ID, tok.TokenID); Assert.Equal("alpha-123_", tok.StringWithoutQuotes); ; @@ -216,9 +224,10 @@ public void TestAlphaNumDashIdStartsWithUnderscore() var lexerRes = LexerBuilder.BuildLexer(new BuildResult>()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; - var r = lexer.Tokenize("_alpha-123_").ToList(); - Assert.Equal(2, r.Count); - var tok = r[0]; + var r = lexer.Tokenize("_alpha-123_"); + Assert.True(r.IsOk); + Assert.Equal(2, r.Tokens.Count); + var tok = r.Tokens[0]; Assert.Equal(AlphaNumDashId.ID, tok.TokenID); Assert.Equal("_alpha-123_", tok.StringWithoutQuotes); ; @@ -231,14 +240,33 @@ public void TestAlphaNumId() var lexerRes = LexerBuilder.BuildLexer(new BuildResult>()); Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; - var r = lexer.Tokenize("alpha123").ToList(); - Assert.Equal(2, r.Count); - var tok = r[0]; + var r = lexer.Tokenize("alpha123"); + Assert.True(r.IsOk); + Assert.Equal(2, r.Tokens.Count); + var tok = r.Tokens[0]; Assert.Equal(AlphaNumId.ID, tok.TokenID); Assert.Equal("alpha123", tok.StringWithoutQuotes); ; } + [Fact] + public void TestStringDelimiters() + { + var lexerRes = LexerBuilder.BuildLexer(new BuildResult>()); + Assert.True(lexerRes.IsOk); + var lexer = lexerRes.Result; + var r = lexer.Tokenize("'that''s it'"); + Assert.True(r.IsOk); + Assert.NotNull(r.Tokens); + Assert.NotEmpty(r.Tokens); + Assert.Equal(2,r.Tokens.Count); + var tok = r.Tokens[0]; + + Assert.Equal("'that's it'",tok.Value); + Assert.Equal("that's it",tok.StringWithoutQuotes); + + } + [Fact] public void TestBadEmptyStringDelimiter() { @@ -291,9 +319,10 @@ public void TestDefaultQuotedString() var lexer = lexerRes.Result; var source = "hello \\\"world "; var expected = "hello \"world "; - var r = lexer.Tokenize($"\"{source}\"").ToList(); - Assert.Equal(2, r.Count); - var tok = r[0]; + var r = lexer.Tokenize($"\"{source}\""); + Assert.True(r.IsOk); + Assert.Equal(2, r.Tokens.Count); + var tok = r.Tokens[0]; Assert.Equal(DefaultQuotedString.DefaultString, tok.TokenID); Assert.Equal(expected, tok.StringWithoutQuotes); } @@ -306,9 +335,10 @@ public void TestDoubleQuotedString() var lexer = lexerRes.Result; var source = "hello \\\"world "; var expected = "hello \"world "; - var r = lexer.Tokenize($"\"{source}\"").ToList(); - Assert.Equal(2, r.Count); - var tok = r[0]; + var r = lexer.Tokenize($"\"{source}\""); + Assert.True(r.IsOk); + Assert.Equal(2, r.Tokens.Count); + var tok = r.Tokens[0]; Assert.Equal(DoubleQuotedString.DoubleString, tok.TokenID); Assert.Equal(expected, tok.StringWithoutQuotes); } @@ -323,24 +353,28 @@ public void TestExtensions() var lexer = lexerRes.Result as GenericLexer; Assert.NotNull(lexer); - var tokens = lexer.Tokenize("20.02.2018 3.14").ToList(); - Assert.Equal(3, tokens.Count); - Assert.Equal(Extensions.DATE, tokens[0].TokenID); - Assert.Equal("20.02.2018", tokens[0].Value); - Assert.Equal(Extensions.DOUBLE, tokens[1].TokenID); - Assert.Equal("3.14", tokens[1].Value); - - tokens = lexer.Tokenize("'that''s it'").ToList(); - Assert.Equal(2, tokens.Count); - var tok = tokens[0]; + var r = lexer.Tokenize("20.02.2018 3.14"); + Assert.True(r.IsOk); + + Assert.Equal(3, r.Tokens.Count); + Assert.Equal(Extensions.DATE, r.Tokens[0].TokenID); + Assert.Equal("20.02.2018", r.Tokens[0].Value); + Assert.Equal(Extensions.DOUBLE, r.Tokens[1].TokenID); + Assert.Equal("3.14", r.Tokens[1].Value); + + r = lexer.Tokenize("'that''s it'"); + Assert.True(r.IsOk); + Assert.Equal(2, r.Tokens.Count); + var tok = r.Tokens[0]; Assert.Equal(Extensions.CHAINE, tok.TokenID); - Assert.Equal("'that's it'", tokens[0].Value); + Assert.Equal("'that's it'", tok.Value); - tokens = lexer.Tokenize("'et voilà'").ToList(); - Assert.Equal(2, tokens.Count); - tok = tokens[0]; + r = lexer.Tokenize("'et voilà'"); + Assert.True(r.IsOk); + Assert.Equal(2, r.Tokens.Count); + tok = r.Tokens[0]; Assert.Equal(Extensions.CHAINE, tok.TokenID); - Assert.Equal("'et voilà'", tokens[0].Value); + Assert.Equal("'et voilà'", tok.Value); } [Fact] @@ -352,16 +386,10 @@ public void TestExtensionsPreconditionFailure() var lexer = lexerRes.Result as GenericLexer; Assert.NotNull(lexer); - var err = Assert.Throws(() => lexer.Tokenize("0.0.2018")); - var lexErr = err as LexerException; - Assert.Equal(0,lexErr.Error.Line); - Assert.Equal(3,lexErr.Error.Column); - Assert.Equal('.',lexErr.Error.UnexpectedChar); - ; - - - - + var r = lexer.Tokenize("0.0.2018"); + Assert.Equal(0,r.Error.Line); + Assert.Equal(3,r.Error.Column); + Assert.Equal('.',r.Error.UnexpectedChar); } [Fact] @@ -371,8 +399,9 @@ public void TestLexerError() Assert.False(lexerRes.IsError); var lexer = lexerRes.Result; var source = "hello world 2 + 2 "; - var errException = Assert.Throws(() => lexer.Tokenize(source).ToList()); - var error = errException.Error; + var r = lexer.Tokenize(source); + Assert.True(r.IsError); + var error = r.Error; Assert.Equal(0, error.Line); Assert.Equal(13, error.Column); Assert.Equal('2', error.UnexpectedChar); @@ -390,13 +419,14 @@ public void TestManyString() var string2 = "'that''s it'"; var expectString2 = "'that's it'"; var source1 = $"{string1} {string2}"; - var r = lexer.Tokenize(source1).ToList(); - Assert.Equal(3, r.Count); - var tok1 = r[0]; + var r = lexer.Tokenize(source1); + Assert.True(r.IsOk); + Assert.Equal(3, r.Tokens.Count); + var tok1 = r.Tokens[0]; Assert.Equal(ManyString.STRING, tok1.TokenID); Assert.Equal(expectString1, tok1.Value); - var tok2 = r[1]; + var tok2 = r.Tokens[1]; Assert.Equal(ManyString.STRING, tok2.TokenID); Assert.Equal(expectString2, tok2.Value); } @@ -408,13 +438,18 @@ public void TestSelfEscapedString() Assert.False(lexerRes.IsError); var lexer = lexerRes.Result as GenericLexer; Assert.NotNull(lexer); - var tokens = lexer.Tokenize("'that''s it'").ToList(); + var r = lexer.Tokenize("'that''s it'"); + + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(2, tokens.Count); var tok = tokens[0]; Assert.Equal(SelfEscapedString.STRING, tok.TokenID); Assert.Equal("'that's it'", tokens[0].Value); - tokens = lexer.Tokenize("'et voilà'").ToList(); + r = lexer.Tokenize("'et voilà'"); + Assert.True(r.IsOk); + tokens = r.Tokens; Assert.Equal(2, tokens.Count); tok = tokens[0]; Assert.Equal(SelfEscapedString.STRING, tok.TokenID); @@ -429,9 +464,10 @@ public void TestSingleQuotedString() var lexer = lexerRes.Result; var source = "hello \\'world "; var expected = "hello 'world "; - var r = lexer.Tokenize($"'{source}'").ToList(); - Assert.Equal(2, r.Count); - var tok = r[0]; + var r = lexer.Tokenize($"'{source}'"); + Assert.True(r.IsOk); + Assert.Equal(2, r.Tokens.Count); + var tok = r.Tokens[0]; Assert.Equal(SingleQuotedString.SingleString, tok.TokenID); Assert.Equal(expected, tok.StringWithoutQuotes); } @@ -445,7 +481,9 @@ public void TestTokenCallbacks() CallBacksBuilder.BuildCallbacks(lexer); - var tokens = lexer.Tokenize("aaa bbb").ToList(); + var r = lexer.Tokenize("aaa bbb"); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(3, tokens.Count); Assert.Equal("AAA", tokens[0].Value); Assert.Equal("BBB", tokens[1].Value); @@ -458,7 +496,9 @@ public void TestIssue106() var res = LexerBuilder.BuildLexer(new BuildResult>()); Assert.False(res.IsError); var lexer = res.Result as GenericLexer; - var tokens = lexer.Tokenize("1.").ToList(); + var r = lexer.Tokenize("1."); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.NotNull(tokens); Assert.Equal(2, tokens.Count); var token = tokens[0]; @@ -472,30 +512,25 @@ public void TestIssue114() var res = LexerBuilder.BuildLexer(new BuildResult>()); Assert.False(res.IsError); var lexer = res.Result as GenericLexer; - var error = Assert.Throws(() => - { - lexer?.Tokenize("// /&").ToList(); - }); - Assert.Equal('&', error.Error.UnexpectedChar); - - error = Assert.Throws(() => - { - lexer?.Tokenize("/&").ToList(); - }); - - Assert.Equal('&', error.Error.UnexpectedChar); - - error = Assert.Throws(() => - { - lexer?.Tokenize("&/").ToList(); - }); - Assert.Equal('&', error.Error.UnexpectedChar); - - error = Assert.Throws(() => - { - lexer?.Tokenize("// &").ToList(); - }); - Assert.Equal('&', error.Error.UnexpectedChar); + + var r = lexer?.Tokenize("// /&"); + Assert.True(r.IsError); + + Assert.Equal('&', r.Error.UnexpectedChar); + + r = lexer?.Tokenize("/&"); + + + Assert.Equal('&', r.Error.UnexpectedChar); + + r = lexer?.Tokenize("&/"); + Assert.True(r.IsError); + + Assert.Equal('&', r.Error.UnexpectedChar); + + r = lexer?.Tokenize("// &"); + Assert.True(r.IsError); + Assert.Equal('&', r.Error.UnexpectedChar); } } } diff --git a/ParserTests/lexer/LexerTests.cs b/ParserTests/lexer/LexerTests.cs index 6a5525f5..3e76eba3 100644 --- a/ParserTests/lexer/LexerTests.cs +++ b/ParserTests/lexer/LexerTests.cs @@ -33,7 +33,7 @@ public void TestDoubleJsonLexing() var lexer = GetJsonLexer(); var tokens = lexer.Tokenize("1.68"); Assert.NotNull(tokens); - var tokenList = tokens.ToList(); + var tokenList = tokens.Tokens; Assert.NotEmpty(tokenList); var token = tokenList[0]; Assert.NotNull(token); @@ -45,7 +45,9 @@ public void TestMultiLineExpressionLexing() { var lexer = GetExpressionLexer(); var expr = "1 + 2 \n* 3"; - var tokens = lexer.Tokenize(expr).ToList(); + var r = lexer.Tokenize(expr); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(6, tokens.Count); var expectedTokensID = new List { @@ -78,7 +80,9 @@ public void TestMultiLineJsonLexing() var json = "{ \"propi\": 12 \n" + ", \"props\":\"val\" }"; var lexer = GetJsonLexer(); - var tokens = lexer.Tokenize(json).ToList(); + var r = lexer.Tokenize(json); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(10, tokens.Count); var expectedTokensID = new List { @@ -114,7 +118,9 @@ public void TestSingleLineExpressionLexing() { var lexer = GetExpressionLexer(); var expr = "1 + 2 * 3"; - var tokens = lexer.Tokenize(expr).ToList(); + var r = lexer.Tokenize(expr); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(6, tokens.Count); var expectedTokensID = new List { @@ -139,7 +145,9 @@ public void TestSingleLineJsonLexing() { var json = "{ \"propi\": 12 , \"props\":\"val\" }"; var lexer = GetJsonLexer(); - var tokens = lexer.Tokenize(json).ToList(); + var r = lexer.Tokenize(json); + Assert.True(r.IsOk); + var tokens = r.Tokens; Assert.Equal(10, tokens.Count); var expectedTokensID = new List { diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs index 6f89af6b..9c62e3ff 100644 --- a/samples/ParserExample/Program.cs +++ b/samples/ParserExample/Program.cs @@ -289,17 +289,25 @@ private static void testJSONLexer() var lexer = new JSONLexer(); var sw = new Stopwatch(); sw.Start(); - var tokens = lexer.Tokenize(source); - sw.Stop(); - Console.WriteLine($"hard coded lexer {tokens.Count()} tokens in {sw.ElapsedMilliseconds}ms"); - var sw2 = new Stopwatch(); - var start = DateTime.Now.Millisecond; - sw2.Start(); - tokens = parser.Result.Lexer.Tokenize(source).ToList(); - sw2.Stop(); - var end = DateTime.Now.Millisecond; - Console.WriteLine($"old lexer {tokens.Count()} tokens in {sw2.ElapsedMilliseconds}ms / {end - start}ms"); - + var lexresult = lexer.Tokenize(source); + if (lexresult.IsOk) + { + var tokens = lexresult.Tokens; + sw.Stop(); + Console.WriteLine($"hard coded lexer {tokens.Count()} tokens in {sw.ElapsedMilliseconds}ms"); + var sw2 = new Stopwatch(); + var start = DateTime.Now.Millisecond; + sw2.Start(); + lexresult = parser.Result.Lexer.Tokenize(source); + if (lexresult.IsOk) + { + tokens = lexresult.Tokens; + sw2.Stop(); + var end = DateTime.Now.Millisecond; + Console.WriteLine( + $"old lexer {tokens.Count()} tokens in {sw2.ElapsedMilliseconds}ms / {end - start}ms"); + } + } ; } @@ -375,11 +383,15 @@ public static void TestTokenCallBacks() var lexer = res.Result as GenericLexer; CallBacksBuilder.BuildCallbacks(lexer); - var tokens = lexer.Tokenize("aaa bbb").ToList(); - ; - foreach (var token in tokens) + var r = lexer.Tokenize("aaa bbb"); + if (r.IsOk) { - Console.WriteLine($"{token.TokenID} - {token.Value}"); + var tokens = r.Tokens; + ; + foreach (var token in tokens) + { + Console.WriteLine($"{token.TokenID} - {token.Value}"); + } } } diff --git a/samples/jsonparser/JSONLexer.cs b/samples/jsonparser/JSONLexer.cs index 4a0c01e7..961fdae6 100644 --- a/samples/jsonparser/JSONLexer.cs +++ b/samples/jsonparser/JSONLexer.cs @@ -11,12 +11,12 @@ public void AddDefinition(TokenDefinition tokenDefinition) { } - public IEnumerable> Tokenize(string source) + public LexerResult Tokenize(string source) { return Tokenize(new ReadOnlyMemory(source.ToCharArray())); } - public IEnumerable> Tokenize(ReadOnlyMemory source) + public LexerResult Tokenize(ReadOnlyMemory source) { var tokens = new List>(); var position = 0; @@ -204,7 +204,7 @@ public IEnumerable> Tokenize(ReadOnlyMemory source) } - return tokens; + return new LexerResult(tokens); } } } \ No newline at end of file diff --git a/sly/lexer/GenericLexer.cs b/sly/lexer/GenericLexer.cs index 6e8c4814..e7592349 100644 --- a/sly/lexer/GenericLexer.cs +++ b/sly/lexer/GenericLexer.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using sly.buildresult; using sly.lexer.fsm; namespace sly.lexer @@ -92,15 +93,23 @@ public void AddCallBack(IN token, Func, Token> callback) CallBacks[token] = callback; } - public void AddDefinition(TokenDefinition tokenDefinition) - { - throw new NotImplementedException(); - } + public void AddDefinition(TokenDefinition tokenDefinition) {} - public IEnumerable> Tokenize(string source) + public LexerResult Tokenize(string source) { + LexerResult result = null; var tokens = new List>(); - var r = LexerFsm.Run(source, 0); + FSMMatch r = null; + + r = LexerFsm.Run(source, 0); + if (!r.IsSuccess && !r.IsEOS) + { + var resultPosition = r.Result.Position; + LexicalError error = + new LexicalError(resultPosition.Line, resultPosition.Column, r.Result.CharValue); + return new LexerResult(error); + } + while (r.IsSuccess) { var transcoded = Transcode(r); @@ -111,17 +120,26 @@ public IEnumerable> Tokenize(string source) } tokens.Add(transcoded); + r = LexerFsm.Run(source); - + if (!r.IsSuccess && !r.IsEOS) + { + var resultPosition = r.Result.Position; + LexicalError error = + new LexicalError(resultPosition.Line, resultPosition.Column, r.Result.CharValue); + return new LexerResult(error); + } + + if (r.IsSuccess && r.Result.IsComment) ConsumeComment(r.Result, source); } - + var eos = new Token(); var prev = tokens.Last(); eos.Position = new TokenPosition(prev.Position.Index + 1, prev.Position.Line, prev.Position.Column + prev.Value.Length); tokens.Add(eos); - return tokens; + return new LexerResult(tokens); } diff --git a/sly/lexer/ILexer.cs b/sly/lexer/ILexer.cs index a105213a..eb33a090 100644 --- a/sly/lexer/ILexer.cs +++ b/sly/lexer/ILexer.cs @@ -1,10 +1,9 @@ -using System.Collections.Generic; - + namespace sly.lexer { - public interface ILexer + public interface ILexer where T : struct { void AddDefinition(TokenDefinition tokenDefinition); - IEnumerable> Tokenize(string source); + LexerResult Tokenize(string source); } } \ No newline at end of file diff --git a/sly/lexer/Lexer.cs b/sly/lexer/Lexer.cs index 6b5a8008..b46d48d1 100644 --- a/sly/lexer/Lexer.cs +++ b/sly/lexer/Lexer.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Linq; +using sly.buildresult; namespace sly.lexer { @@ -7,7 +8,7 @@ namespace sly.lexer /// T is the token type /// /// T is the enum Token type - public class Lexer : ILexer + public class Lexer : ILexer where T : struct { private readonly IList> tokenDefinitions = new List>(); @@ -17,8 +18,10 @@ public void AddDefinition(TokenDefinition tokenDefinition) } - public IEnumerable> Tokenize(string source) + public LexerResult Tokenize(string source) { + List> tokens = new List>(); + var currentIndex = 0; //List> tokens = new List>(); var currentLine = 1; @@ -49,7 +52,7 @@ public IEnumerable> Tokenize(string source) if (matchedDefinition == null) { - throw new LexerException(new LexicalError(currentLine, currentColumn, source[currentIndex])); + return new LexerResult(new LexicalError(currentLine, currentColumn, source[currentIndex])); } var value = source.Substring(currentIndex, matchLength); @@ -64,7 +67,7 @@ public IEnumerable> Tokenize(string source) { previousToken = new Token(matchedDefinition.TokenID, value, new TokenPosition(currentIndex, currentLine, currentColumn)); - yield return previousToken; + tokens.Add(previousToken); } currentIndex += matchLength; @@ -78,7 +81,8 @@ public IEnumerable> Tokenize(string source) previousToken.Position.Column + previousToken.Value.Length); - yield return eos; + tokens.Add(eos); + return new LexerResult(tokens); } } } \ No newline at end of file diff --git a/sly/lexer/LexerBuilder.cs b/sly/lexer/LexerBuilder.cs index bd5f8fde..1ce1a6bb 100644 --- a/sly/lexer/LexerBuilder.cs +++ b/sly/lexer/LexerBuilder.cs @@ -33,7 +33,7 @@ public static class EnumHelper public class LexerBuilder { - public static Dictionary> GetLexemes(BuildResult> result) + public static Dictionary> GetLexemes(BuildResult> result) where IN: struct { var values = Enum.GetValues(typeof(IN)); @@ -318,7 +318,7 @@ private static bool IsGenericLexer(Dictionary> att } - private static Dictionary> GetCommentsAttribute(BuildResult> result) + private static Dictionary> GetCommentsAttribute(BuildResult> result) where IN : struct { var values = Enum.GetValues(typeof(IN)); diff --git a/sly/lexer/LexerException.cs b/sly/lexer/LexerException.cs deleted file mode 100644 index bda74047..00000000 --- a/sly/lexer/LexerException.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace sly.lexer -{ - public class LexerException : Exception - { - public LexerException(LexicalError error) - { - Error = error; - } - - public LexicalError Error { get; set; } - } -} \ No newline at end of file diff --git a/sly/lexer/LexerResult.cs b/sly/lexer/LexerResult.cs new file mode 100644 index 00000000..6be9e954 --- /dev/null +++ b/sly/lexer/LexerResult.cs @@ -0,0 +1,27 @@ +using System.Collections.Generic; +namespace sly.lexer +{ + public class LexerResult where IN : struct + { + public bool IsError { get; set; } + + public bool IsOk => !IsError; + + public LexicalError Error { get; } + + public List> Tokens { get; set; } + + public LexerResult(List> tokens) + { + IsError = false; + Tokens = tokens; + } + + public LexerResult(LexicalError error) + { + IsError = true; + Error = error; + } + + } +} \ No newline at end of file diff --git a/sly/lexer/fsm/FSMLexer.cs b/sly/lexer/fsm/FSMLexer.cs index 30ce0281..44374d23 100644 --- a/sly/lexer/fsm/FSMLexer.cs +++ b/sly/lexer/fsm/FSMLexer.cs @@ -262,8 +262,12 @@ public FSMMatch Run( ReadOnlyMemory source, int start) else { if (!successes.Any() && CurrentPosition < source.Length) - throw new LexerException(new LexicalError(CurrentLine, CurrentColumn, - source.At(CurrentPosition))); + { + var errorChar = source.Slice(CurrentPosition, 1); + var errorPosition = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn); + var ko = new FSMMatch(false, default(N), errorChar, errorPosition, -1); + return ko; + } } } } diff --git a/sly/lexer/fsm/FSMMatch.cs b/sly/lexer/fsm/FSMMatch.cs index 7813e8df..378f433b 100644 --- a/sly/lexer/fsm/FSMMatch.cs +++ b/sly/lexer/fsm/FSMMatch.cs @@ -11,6 +11,8 @@ public class FSMMatch public Dictionary Properties { get; set; } public bool IsSuccess { get; set; } + + public bool IsEOS { get; set; } public Token Result { get; set; } @@ -18,11 +20,14 @@ public class FSMMatch public FSMMatch(bool success) { IsSuccess = success; + if (!IsSuccess) + { + IsEOS = true; + } } public FSMMatch(bool success, N result, string value, TokenPosition position,int nodeId) : this(success,result,new ReadOnlyMemory(value.ToCharArray()),position,nodeId ) { - } public FSMMatch(bool success, N result, ReadOnlyMemory value, TokenPosition position, int nodeId) @@ -30,8 +35,11 @@ public FSMMatch(bool success, N result, ReadOnlyMemory value, TokenPositio Properties = new Dictionary(); IsSuccess = success; NodeId = nodeId; + IsEOS = false; Result = new Token(result, value, position); } + + } diff --git a/sly/parser/parser/Parser.cs b/sly/parser/parser/Parser.cs index 2249acaa..4d7d939e 100644 --- a/sly/parser/parser/Parser.cs +++ b/sly/parser/parser/Parser.cs @@ -56,43 +56,45 @@ public Parser(ISyntaxParser syntaxParser, SyntaxTreeVisitor vi { return ParseWithContext(source,new NoContext(),startingNonTerminal); } - - + + public ParseResult ParseWithContext(string source, object context, string startingNonTerminal = null) { ParseResult result = null; - try - { - IList> tokens = Lexer.Tokenize(source).ToList(); - var position = 0; - var tokensWithoutComments = new List>(); - for (var i = 0; i < tokens.Count; i++) - { - var token = tokens[i]; - if (!token.IsComment) - { - token.PositionInTokenFlow = position; - tokensWithoutComments.Add(token); - position++; - } - } - result = ParseWithContext(tokensWithoutComments, context, startingNonTerminal); - } - catch (LexerException e) + var lexingResult = Lexer.Tokenize(source); + if (lexingResult.IsError) { result = new ParseResult(); result.IsError = true; result.Errors = new List(); - result.Errors.Add(e.Error); + result.Errors.Add(lexingResult.Error); + return result; } + var tokens = lexingResult.Tokens; + var position = 0; + var tokensWithoutComments = new List>(); + for (var i = 0; i < tokens.Count; i++) + { + var token = tokens[i]; + if (!token.IsComment) + { + token.PositionInTokenFlow = position; + tokensWithoutComments.Add(token); + position++; + } + } + + result = ParseWithContext(tokensWithoutComments, context, startingNonTerminal); + + return result; } - - + + public ParseResult ParseWithContext(IList> tokens, object parsingContext = null, string startingNonTerminal = null) { var result = new ParseResult();