diff --git a/.github/workflows/.dotnetcore.yml.un~ b/.github/workflows/.dotnetcore.yml.un~ new file mode 100644 index 00000000..35f1fb25 Binary files /dev/null and b/.github/workflows/.dotnetcore.yml.un~ differ diff --git a/.github/workflows/dotnetcore.yml b/.github/workflows/dotnetcore.yml index ed2b58f2..eafd93a7 100644 --- a/.github/workflows/dotnetcore.yml +++ b/.github/workflows/dotnetcore.yml @@ -6,6 +6,7 @@ on: - dev - tech/actions - master + - feature/implicit-tokens pull_request: branches: - dev diff --git a/.github/workflows/dotnetcore.yml~ b/.github/workflows/dotnetcore.yml~ new file mode 100644 index 00000000..ec120cfc --- /dev/null +++ b/.github/workflows/dotnetcore.yml~ @@ -0,0 +1,103 @@ +name: .NET Core + +on: + push: + branches: + - dev + - tech/actions + - master + pull_request: + branches: + - dev +permissions: + pull-requests: write +jobs: + build: + env: + TESTS_PROJECT: 'ParserTests/ParserTests.csproj' # path to test project or solution + PUBLISH_NUGET: true # if true a nuget will be published on version change + RUN_TESTS: true # if true tests are run and coverage data is published to coveralls and a coverage report is produced. + MAIN_CSPROJ: 'sly/sly.csproj' # main project (for nuget packaging) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - uses: actions/checkout@v2 + - name: Setup .NET Core + uses: actions/setup-dotnet@v1 + with: + dotnet-version: 3.1.101 + - name: Clean artifacts and nugets + run: dotnet clean --configuration Release && dotnet nuget locals all --clear + - name: Build with dotnet + run: dotnet build --configuration Release + - name: Test with dotnet + uses: b3b00/coverlet-action@1.2.2 + id: 'coverlet' + if: env.RUN_TESTS + with: + testProject: ${{env.TESTS_PROJECT}} + output: 'lcov.info' + threshold: 80 + outputFormat: 'lcov' + excludes: '[program]*,[expressionParser]*,[jsonparser]*,[while]*,[indentedWhile]*,[SimpleExpressionParser]*,[GenericLexerWithCallbacks]*,[indented]*' + - name: coveralls + uses: coverallsapp/github-action@v1.1.1 + if: matrix.os == 'windows-latest' && env.RUN_TESTS + with: + github-token: ${{secrets.GITHUB_TOKEN }} + path-to-lcov: ${{steps.coverlet.outputs.coverageFile}} + - name: ReportGenerator + uses: danielpalme/ReportGenerator-GitHub-Action@4.8.12 + with: + reports: ${{steps.coverlet.outputs.coverageFile}} + targetdir: 'coveragereport' + reporttypes: 'HtmlInline;MarkdownSummary' + verbosity: 'Info' # The verbosity level of the log messages. Values: Verbose, Info, Warning, Error, Off + tag: '${{ github.run_number }}_${{ github.run_id }}' +# - name: Publish coverage summary +# uses: marocchino/sticky-pull-request-comment@v2 +# with: +# path: coveragereport/Summary.md + - name: publish nuget + if: matrix.os == 'windows-latest' && env.PUBLISH_NUGET + id: publish_nuget + uses: brandedoutcast/publish-nuget@v2.5.2 + with: + VERSION_REGEX: (.*)<\/version> + PROJECT_FILE_PATH: ${{env.MAIN_CSPROJ}} + NUGET_KEY: ${{secrets.NUGET_KEY}} + VERSION_FILE_PATH: ${{env.MAIN_CSPROJ}} + - name: Create Release + if: ${{ success() && matrix.os == 'windows-latest' && steps.publish_nuget.outputs.VERSION != '' && steps.publish_nuget.outputs.VERSION != null }} + id: create_release + uses: actions/create-release@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.publish_nuget.outputs.VERSION }} + release_name: Release ${{ steps.publish_nuget.outputs.VERSION }} + draft: false + prerelease: false + - name: Upload Release Asset + if: ${{ success() && matrix.os == 'windows-latest' && steps.create_release.outputs.upload_url != '' && steps.create_release.outputs.upload_url != null }} + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ steps.publish_nuget.outputs.PACKAGE_PATH }} + asset_name: ${{ steps.publish_nuget.outputs.PACKAGE_NAME }} + asset_content_type: application/zip + - name: refresh coverage badge + uses: fjogeleit/http-request-action@master + with: + url: https://camo.githubusercontent.com/12c4fcb3b21fbb2a725fc61449fb1b91e972c4c8a2baaf5904936d8e334bdbe8/68747470733a2f2f636f766572616c6c732e696f2f7265706f732f6769746875622f62336230302f63736c792f62616467652e7376673f6272616e63683d64657626736572766963653d676974687562 + method: PURGE + - name: refresh nuget badge + uses: fjogeleit/http-request-action@master + with: + url: https://camo.githubusercontent.com/b0c8ccfcb3256380ae37c312e789e6282143295fa495361ea5c2dbe2169bff8c/68747470733a2f2f696d672e736869656c64732e696f2f6e756765742f762f736c792e737667 + method: PURGE diff --git a/ParserTests/ImplicitTokensExpressionParser.cs b/ParserTests/ImplicitTokensExpressionParser.cs new file mode 100644 index 00000000..c566e779 --- /dev/null +++ b/ParserTests/ImplicitTokensExpressionParser.cs @@ -0,0 +1,108 @@ +using System; +using expressionparser; +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests +{ + public class ImplicitTokensExpressionParser + { + [Production("primary: DOUBLE")] + [Operand] + public double Primary(Token doubleToken) + { + return doubleToken.DoubleValue; + } + + [Operand] + [Production("primary : 'bozzo'[d]")] + public double Bozzo() + { + return 42.0; + } + + [Operand] + [Production("primary : TEST[d]")] + public double Test() + { + return 0.0; + } + + + [Operation("'+'", Affix.InFix, Associativity.Left, 10)] + [Operation("'-'", Affix.InFix, Associativity.Left, 10)] + public double BinaryTermExpression(double left, Token operation, double right) + { + switch (operation.Value) + { + case "+" : return left + right; + case "-" : return left - right; + default : throw new InvalidOperationException($"that is not possible ! {operation.Value} is not a valid operation"); + } + return 0; + } + + [Operation((int) ImplicitTokensTokens.TIMES, Affix.InFix, Associativity.Left, 50)] + [Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)] + public double BinaryFactorExpression(double left, Token operation, double right) + { + double result = 0; + switch (operation.TokenID) + { + case ImplicitTokensTokens.TIMES: + { + result = left * right; + break; + } + case ImplicitTokensTokens.DIVIDE: + { + result = left / right; + break; + } + } + + return result; + } + + + [Operation("'-'", Affix.PreFix, Associativity.Right, 100)] + public double PreFixExpression(Token operation, double value) + { + return -value; + } + + + public double Expression(double left, Token operatorToken, double right) + { + double result = 0.0; + + + switch (operatorToken.StringWithoutQuotes) + { + case "+": + { + result = left + right; + break; + } + case "-": + { + result = left - right; + break; + } + } + + return result; + } + + + [Production("expression : primary ")] + public double Simple(double value) + { + return value; + } + + + + + } +} \ No newline at end of file diff --git a/ParserTests/ImplicitTokensParser.cs b/ParserTests/ImplicitTokensParser.cs new file mode 100644 index 00000000..5c5660e7 --- /dev/null +++ b/ParserTests/ImplicitTokensParser.cs @@ -0,0 +1,63 @@ +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests +{ + public class ImplicitTokensParser + { + [Production("primary: DOUBLE")] + public double Primary(Token doubleToken) + { + return doubleToken.DoubleValue; + } + + [Production("primary : 'bozzo'[d]")] + public double Bozzo() + { + return 42.0; + } + + [Production("primary : TEST[d]")] + public double Test() + { + return 0.0; + } + + + [Production("expression : primary ['+' | '-'] expression")] + + + public double Expression(double left, Token operatorToken, double right) + { + double result = 0.0; + + + switch (operatorToken.StringWithoutQuotes) + { + case "+": + { + result = left + right; + break; + } + case "-": + { + result = left - right; + break; + } + } + + return result; + } + + + [Production("expression : primary ")] + public double Simple(double value) + { + return value; + } + + + + + } +} \ No newline at end of file diff --git a/ParserTests/ImplicitTokensTests.cs b/ParserTests/ImplicitTokensTests.cs new file mode 100644 index 00000000..54d1e6c4 --- /dev/null +++ b/ParserTests/ImplicitTokensTests.cs @@ -0,0 +1,73 @@ +using System.IO; +using sly.buildresult; +using sly.parser; +using sly.parser.generator; +using sly.parser.generator.visitor; +using Xunit; + +namespace ParserTests +{ + public class ImplicitTokensTests + { + private Parser Parser; + + private BuildResult> BuildParser() + { + var parserInstance = new ImplicitTokensParser(); + var builder = new ParserBuilder(); + var result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "expression"); + return result; + } + + private BuildResult> BuildExpressionParser() + { + var parserInstance = new ImplicitTokensExpressionParser(); + var builder = new ParserBuilder(); + var result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, nameof(ImplicitTokensExpressionParser)+"_expressions"); + return result; + } + + [Fact] + public void BuildParserTest() + { + var parser = BuildParser(); + Assert.True(parser.IsOk); + Assert.NotNull(parser.Result); + var r = parser.Result.Parse("2.0 - 2.0 + bozzo + Test"); + Assert.True(r.IsOk); + // grammar is left associative so expression really is + // (2.0 - (2.0 + (bozzo + Test))) = 2 - ( 2 + (42 + 0)) = 2 - (2 + 42) = 2 - 44 = -42 + Assert.Equal(-42.0,r.Result); + } + + [Fact] + public void BuildExpressionParserTest() + { + var parser = BuildExpressionParser(); + Assert.True(parser.IsOk); + Assert.NotNull(parser.Result); + var r = parser.Result.Parse("2.0 - 2.0 + bozzo + Test"); + var tree = r.SyntaxTree; + var graphviz = new GraphVizEBNFSyntaxTreeVisitor(); + var dump = tree.Dump("\t"); + // File.Delete(@"c:\temp\tree.txt"); + // File.WriteAllText(@"c:\temp\tree.txt",dump); + // + var json = $@"{{ +{tree.ToJson()} +}}"; + // File.Delete(@"c:\temp\tree.json"); + // File.WriteAllText(@"c:\temp\tree.json",json); + // + var root = graphviz.VisitTree(tree); + string graph = graphviz.Graph.Compile(); + // File.Delete("c:\\temp\\tree.dot"); + // File.AppendAllText("c:\\temp\\tree.dot", graph); + Assert.True(r.IsOk); + + + Assert.Equal(2 - 2 + 42 + 0,r.Result); + } + } + +} diff --git a/ParserTests/ImplicitTokensTokens.cs b/ParserTests/ImplicitTokensTokens.cs new file mode 100644 index 00000000..13f5c8ad --- /dev/null +++ b/ParserTests/ImplicitTokensTokens.cs @@ -0,0 +1,28 @@ +using sly.lexer; + +namespace ParserTests +{ + [Lexer(IgnoreWS = true, IgnoreEOL = true)] + public enum ImplicitTokensTokens + { + [MultiLineComment("/*","*/")] + MULTILINECOMMENT = 1, + + [SingleLineComment("//")] + SINGLELINECOMMENT = 2, + + [Lexeme(GenericToken.Identifier, IdentifierType.AlphaNumeric)] + ID = 3, + + [Lexeme(GenericToken.Double, channel:101)] + DOUBLE = 4, + + [Keyword("Test")] + TEST = 5, + [Sugar("*")] + TIMES = 6, + + [Sugar("/")] + DIVIDE = 7 + } +} \ No newline at end of file diff --git a/ParserTests/ParserConfigurationTests.cs b/ParserTests/ParserConfigurationTests.cs index c592ba56..56596ee2 100644 --- a/ParserTests/ParserConfigurationTests.cs +++ b/ParserTests/ParserConfigurationTests.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using expressionparser; using sly.buildresult; @@ -391,14 +392,15 @@ public void TestBadManyArgument() var instance = new BadManyArgParser(); ParserBuilder builder = new ParserBuilder("en"); var result = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "badmanyarg"); + Assert.True(result.IsError); Assert.Equal(4,result.Errors.Count); //"visitor BadReturn for rule badtermarg : A B ; parameter a has incorrect type : expected sly.lexer.Token`1[ParserTests.BadVisitorTokens], found SubBadVisitor" - Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter aArg has incorrect type"))); - Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter bArg has incorrect type"))); - Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter cArg has incorrect type"))); - Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter dArg has incorrect type"))); + Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter aArg has incorrect type") && x.Code == ErrorCodes.PARSER_INCORRECT_VISITOR_PARAMETER_TYPE)); + Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter bArg has incorrect type") && x.Code == ErrorCodes.PARSER_INCORRECT_VISITOR_PARAMETER_TYPE)); + Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter cArg has incorrect type") && x.Code == ErrorCodes.PARSER_INCORRECT_VISITOR_PARAMETER_TYPE)); + Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter dArg has incorrect type") && x.Code == ErrorCodes.PARSER_INCORRECT_VISITOR_PARAMETER_TYPE)); } diff --git a/ParserTests/lexer/GenericLexerTests.cs b/ParserTests/lexer/GenericLexerTests.cs index db0a3c7c..0d92546e 100644 --- a/ParserTests/lexer/GenericLexerTests.cs +++ b/ParserTests/lexer/GenericLexerTests.cs @@ -9,6 +9,7 @@ using sly.parser; using sly.parser.generator; using Xunit; +using Xunit.Abstractions; namespace ParserTests.lexer { @@ -467,6 +468,11 @@ public enum Issue177Regex public class GenericLexerTests { + + public GenericLexerTests() + { + } + [Fact] public void TestEmptyInput() { diff --git a/sly/EnumConverter.cs b/sly/EnumConverter.cs index cfdd7fe4..a0a1e0b3 100644 --- a/sly/EnumConverter.cs +++ b/sly/EnumConverter.cs @@ -21,6 +21,25 @@ public static IN ConvertIntToEnum(int intValue) return default(IN); } + + public static bool IsEnumValue(int intValue) + { + var genericType = typeof(IN); + if (genericType.IsEnum) + foreach (IN value in Enum.GetValues(genericType)) + { + var test = Enum.Parse(typeof(IN), value.ToString()) as Enum; + var val = Convert.ToInt32(test); + if (val == intValue) + { + return true; + } + } + + return false; + } + + public static IN ConvertStringToEnum(string name) where IN : struct { @@ -31,5 +50,11 @@ public static IN ConvertIntToEnum(int intValue) } return token; } + + public static bool IsEnumValue(string name) where IN : struct + { + + return Enum.TryParse(name, out IN token); + } } } \ No newline at end of file diff --git a/sly/lexer/LexerBuilder.cs b/sly/lexer/LexerBuilder.cs index 221aa23a..a7ece483 100644 --- a/sly/lexer/LexerBuilder.cs +++ b/sly/lexer/LexerBuilder.cs @@ -85,12 +85,12 @@ public static class LexerBuilder public static BuildResult> BuildLexer(BuildResult> result, BuildExtension extensionBuilder = null, - string lang = null, LexerPostProcess lexerPostProcess = null) where IN : struct + string lang = null, LexerPostProcess lexerPostProcess = null, IList implicitTokens = null) where IN : struct { var attributes = GetLexemes(result,lang); if (!result.IsError) { - result = Build(attributes, result, extensionBuilder,lang); + result = Build(attributes, result, extensionBuilder,lang, implicitTokens); result.Result.LexerPostProcess = lexerPostProcess; } @@ -99,7 +99,8 @@ public static class LexerBuilder private static BuildResult> Build(Dictionary> attributes, - BuildResult> result, BuildExtension extensionBuilder = null, string lang = null) where IN : struct + BuildResult> result, BuildExtension extensionBuilder = null, string lang = null, + IList implicitTokens = null) where IN : struct { var hasRegexLexemes = IsRegexLexer(attributes); var hasGenericLexemes = IsGenericLexer(attributes); @@ -114,7 +115,7 @@ public static class LexerBuilder { if (hasRegexLexemes) result = BuildRegexLexer(attributes, result,lang); - else if (hasGenericLexemes) result = BuildGenericLexer(attributes, extensionBuilder, result, lang); + else if (hasGenericLexemes) result = BuildGenericLexer(attributes, extensionBuilder, result, lang, implicitTokens); } return result; @@ -149,7 +150,6 @@ private static bool IsGenericLexer(Dictionary> att { foreach (var lexeme in lexemes) { - // TODO ??? var channel = lexeme.Channel.HasValue ? lexeme.Channel.Value : 0; lexer.AddDefinition(new TokenDefinition(tokenID, lexeme.Pattern, channel,lexeme.IsSkippable, lexeme.IsLineEnding)); @@ -258,7 +258,8 @@ private static IEnumerable ParseIdentifierPattern(string pattern) } private static BuildResult> BuildGenericLexer(Dictionary> attributes, - BuildExtension extensionBuilder, BuildResult> result, string lang) where IN : struct + BuildExtension extensionBuilder, BuildResult> result, string lang, + IList implicitTokens = null) where IN : struct { result = CheckStringAndCharTokens(attributes, result, lang); var (config, tokens) = GetConfigAndGenericTokens(attributes); @@ -322,6 +323,7 @@ private static IEnumerable ParseIdentifierPattern(string pattern) AddExtensions(Extensions, extensionBuilder, lexer); var comments = GetCommentsAttribute(result,lang); + if (!result.IsError) { foreach (var comment in comments) @@ -361,6 +363,34 @@ private static IEnumerable ParseIdentifierPattern(string pattern) } } } + + if (implicitTokens != null) + { + foreach (var implicitToken in implicitTokens) + { + var fsmBuilder = lexer.FSMBuilder; + var x = fsmBuilder.Fsm.Run(implicitToken, new LexerPosition()); + if (x.IsSuccess) + { + var t = fsmBuilder.Marks; + var y = fsmBuilder.Marks.FirstOrDefault(k => k.Value == x.NodeId); + if (y.Key == GenericLexer.in_identifier) // implicit keyword + { + var resultx = new BuildResult>(); + result.Errors.AddRange(resultx.Errors); + lexer.AddKeyWord(default(IN), implicitToken, resultx); + ; + } + } + else + { + var resulty = new BuildResult>(); + result.Errors.AddRange(resulty.Errors); + lexer.AddSugarLexem(default(IN), resulty, implicitToken); + } + } + } + } result.Result = lexer; diff --git a/sly/lexer/Token.cs b/sly/lexer/Token.cs index 41cebe6b..912eb302 100644 --- a/sly/lexer/Token.cs +++ b/sly/lexer/Token.cs @@ -113,6 +113,8 @@ public Token Previous(int channelId) public bool IsWhiteSpace { get; set; } public bool IsEOL { get; set; } + + public bool IsImplicit { get; set; } public CommentType CommentType { get; set; } = CommentType.No; @@ -222,6 +224,11 @@ public override string ToString() { value = $"<>"; } + + if (IsImplicit) + { + value = $"[{Value.Replace("\r", "").Replace("\n", "")}]"; + } return $"{value} @{Position} on channel {Channel}"; diff --git a/sly/lexer/TokenChannels.cs b/sly/lexer/TokenChannels.cs index eabc45c2..7cd9615b 100644 --- a/sly/lexer/TokenChannels.cs +++ b/sly/lexer/TokenChannels.cs @@ -23,7 +23,6 @@ public class TokenChannels : IEnumerable> public List> Tokens => GetChannel(Channels.Main).Tokens.Where(x => x != null).ToList(); - public readonly int ChannelId; public TokenChannels() { @@ -54,11 +53,6 @@ public TokenChannel GetChannel(int i) { return _tokenChannels[i]; } - - public void SetChannel(int i, TokenChannel token) - { - _tokenChannels[i] = token; - } public Token this[int index] { @@ -112,48 +106,6 @@ public void Add(Token token) } - public bool ContainsChannel(int channel) => _tokenChannels.ContainsKey(channel); - - - - public Token TokenAt(int index) - { - foreach (var channel in _tokenChannels) - { - var token = TokenInChannelAt(channel.Value, index); - if (token != null) - { - return null; - } - } - return null; - } - - public Token TokenInChannelAt(TokenChannel channel, int index) - { - - if (channel != null) - { - if (index >= 0 && index < channel.Count) - { - return channel[index]; - } - } - return null; - } - - public Token TokenInChannelAt(int channelId, int index) - { - TokenChannel channel = null; - if (TryGet(index, out channel)) - { - if (index >= 0 && index < channel.Count) - { - return channel[index]; - } - } - return null; - } public bool TryGet(int index, out TokenChannel channel) => _tokenChannels.TryGetValue(index, out channel); @@ -169,7 +121,6 @@ IEnumerator IEnumerable.GetEnumerator() public override string ToString() { - var channels = _tokenChannels.Values.OrderBy(x => x.ChannelId); return string.Join("\n", _tokenChannels.Values.Select(x => x.ToString()).ToArray()); } } diff --git a/sly/lexer/fsm/FSMLexerBuilder.cs b/sly/lexer/fsm/FSMLexerBuilder.cs index 3349a4e0..10e3d8e9 100644 --- a/sly/lexer/fsm/FSMLexerBuilder.cs +++ b/sly/lexer/fsm/FSMLexerBuilder.cs @@ -13,7 +13,7 @@ public class FSMLexerBuilder { private int CurrentState; - private readonly Dictionary Marks; + internal readonly Dictionary Marks; public FSMLexerBuilder() diff --git a/sly/parser/generator/EBNFParserBuilder.cs b/sly/parser/generator/EBNFParserBuilder.cs index 986c7d92..667d4e52 100644 --- a/sly/parser/generator/EBNFParserBuilder.cs +++ b/sly/parser/generator/EBNFParserBuilder.cs @@ -38,6 +38,8 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) { configuration = ExtractEbnfParserConfiguration(parserInstance.GetType(), grammarParser); LeftRecursionChecker recursionChecker = new LeftRecursionChecker(); + + // check left recursion. var (foundRecursion, recursions) = LeftRecursionChecker.CheckLeftRecursion(configuration); if (foundRecursion) { @@ -46,7 +48,6 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) I18N.Instance.GetText(I18n,Message.LeftRecursion,recs), ErrorCodes.PARSER_LEFT_RECURSIVE)); return result; - } configuration.StartingRule = rootRule; @@ -71,7 +72,7 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) } var parser = new Parser(I18n,syntaxParser, visitor); parser.Configuration = configuration; - var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess); + var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess, configuration.GetAllImplicitTokenClauses().Select(x => x.ImplicitToken).Distinct().ToList()); if (lexerResult.IsError) { foreach (var lexerResultError in lexerResult.Errors) @@ -81,10 +82,12 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) return result; } else + { parser.Lexer = lexerResult.Result; - parser.Instance = parserInstance; - result.Result = parser; - return result; + parser.Instance = parserInstance; + result.Result = parser; + return result; + } } diff --git a/sly/parser/generator/EbnfToken.cs b/sly/parser/generator/EbnfToken.cs index 496ed4ac..94fb6174 100644 --- a/sly/parser/generator/EbnfToken.cs +++ b/sly/parser/generator/EbnfToken.cs @@ -52,7 +52,10 @@ public enum EbnfTokenGeneric LCROG = 11, [Lexeme(GenericToken.SugarToken,"]")] - RCROG = 12 + RCROG = 12, + + [Lexeme(GenericToken.String, "'","\\")] + STRING = 13, } } \ No newline at end of file diff --git a/sly/parser/generator/ExpressionRulesGenerator.cs b/sly/parser/generator/ExpressionRulesGenerator.cs index bc72848b..0c49a07f 100644 --- a/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/sly/parser/generator/ExpressionRulesGenerator.cs @@ -1,5 +1,4 @@ -using System.Diagnostics.CodeAnalysis; -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -10,38 +9,6 @@ namespace sly.parser.generator { - public class OperationMetaData where T : struct - { - public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, Affix affix, T oper) - { - Precedence = precedence; - Associativity = assoc; - VisitorMethod = method; - OperatorToken = oper; - Affix = affix; - } - - public int Precedence { get; set; } - - public Associativity Associativity { get; set; } - - public MethodInfo VisitorMethod { get; set; } - - public T OperatorToken { get; set; } - - public Affix Affix { get; set; } - - public bool IsBinary => Affix == Affix.InFix; - - public bool IsUnary => Affix != Affix.InFix; - - [ExcludeFromCodeCoverage] - public override string ToString() - { - return $"{OperatorToken} / {Affix} : {Precedence} / {Associativity}"; - } - } - public class ExpressionRulesGenerator where IN : struct { public string I18n { get; set; } @@ -77,15 +44,40 @@ public ExpressionRulesGenerator(string i18n = null) foreach (var attr in attributes) { IN oper = default; + string implicitToken = null; if (attr.IsIntToken) { oper = EnumConverter.ConvertIntToEnum(attr.IntToken); } else if (attr.IsStringToken) { - oper = EnumConverter.ConvertStringToEnum(attr.StringToken); + if (EnumConverter.IsEnumValue(attr.StringToken)) + { + oper = EnumConverter.ConvertStringToEnum(attr.StringToken); + } + else + { + implicitToken = attr.StringToken; + } + } + + + bool isEnumValue = EnumConverter.IsEnumValue(attr.StringToken) || + EnumConverter.IsEnumValue(attr.IntToken); + OperationMetaData operation = null; + if (!isEnumValue && !string.IsNullOrEmpty(implicitToken) && implicitToken.StartsWith("'") && implicitToken.EndsWith("'")) + { + operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, implicitToken); + } + else if (isEnumValue) + { + operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, oper); } - var operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, oper); + else + { + throw new ParserConfigurationException($"bad enum name {attr.StringToken} on Operation definition."); + } + var operations = new List>(); if (operationsByPrecedence.ContainsKey(operation.Precedence)) operations = operationsByPrecedence[operation.Precedence]; @@ -218,7 +210,17 @@ private NonTerminal BuildPrecedenceNonTerminal(string name, string nextName, var InFixOps = operations.Where>(x => x.Affix == Affix.InFix).ToList>(); if (InFixOps.Count > 0) { - var InFixClauses = InFixOps.Select, TerminalClause>(x => new TerminalClause(x.OperatorToken)).ToList>(); + var InFixClauses = InFixOps.Select, TerminalClause>(x => + { + if (x.IsImplicitOperatorToken) + { + return new TerminalClause(x.ImplicitOperatorToken.Substring(1,x.ImplicitOperatorToken.Length-2)); + } + else + { + return new TerminalClause(x.OperatorToken); + } + }).ToList>(); var rule = new Rule { diff --git a/sly/parser/generator/OperationMetaData.cs b/sly/parser/generator/OperationMetaData.cs new file mode 100644 index 00000000..13e4fcf0 --- /dev/null +++ b/sly/parser/generator/OperationMetaData.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.CodeAnalysis; +using System.Reflection; + +namespace sly.parser.generator +{ + public class OperationMetaData where T : struct + { + public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, Affix affix, T oper) + { + Precedence = precedence; + Associativity = assoc; + VisitorMethod = method; + OperatorToken = oper; + Affix = affix; + } + + public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, Affix affix, string oper) + { + Precedence = precedence; + Associativity = assoc; + VisitorMethod = method; + ImplicitOperatorToken = oper; + Affix = affix; + } + + public int Precedence { get; set; } + + public Associativity Associativity { get; set; } + + public MethodInfo VisitorMethod { get; set; } + + public T OperatorToken { get; set; } + + public Affix Affix { get; set; } + + public bool IsBinary => Affix == Affix.InFix; + + public bool IsUnary => Affix != Affix.InFix; + + public bool IsImplicitOperatorToken => !string.IsNullOrEmpty(ImplicitOperatorToken); + + public string ImplicitOperatorToken { get; set; } + + [ExcludeFromCodeCoverage] + public override string ToString() + { + return $"{OperatorToken} / {Affix} : {Precedence} / {Associativity}"; + } + } +} \ No newline at end of file diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index 61735ce1..9b25e324 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -76,13 +76,7 @@ public ParserBuilder() : this(null) var syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule); var visitor = new SyntaxTreeVisitor(configuration, parserInstance); parser = new Parser(I18n,syntaxParser, visitor); - var lexerResult = BuildLexer(extensionBuilder, lexerPostProcess); - parser.Lexer = lexerResult.Result; - if (lexerResult.IsError) - { - result.Errors.AddRange(lexerResult.Errors); - return result; - } + parser.Instance = parserInstance; parser.Configuration = configuration; result.Result = parser; @@ -102,13 +96,33 @@ public ParserBuilder() : this(null) { var expressionResult = parser.BuildExpressionParser(result, rootRule); if (expressionResult.IsError) result.AddErrors(expressionResult.Errors); + + + result.Result.Configuration = expressionResult.Result; - + + var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess, result.Result.Configuration.GetAllImplicitTokenClauses().Select(x => x.ImplicitToken).Distinct().ToList()); + if (lexerResult.IsError) + { + foreach (var lexerResultError in lexerResult.Errors) + { + result.AddError(lexerResultError); + } + } + else + { + parser.Lexer = lexerResult.Result; + parser.Instance = parserInstance; + result.Result = parser; + + } + result = CheckParser(result); if (result.IsError) { result.Result = null; } + return result; } else { @@ -167,9 +181,12 @@ public ParserBuilder() : this(null) protected virtual BuildResult> BuildLexer(BuildExtension extensionBuilder = null, - LexerPostProcess lexerPostProcess = null) + LexerPostProcess lexerPostProcess = null, IList implicitTokens = null) { - var lexer = LexerBuilder.BuildLexer(new BuildResult>(), extensionBuilder, I18n, lexerPostProcess); + + + var lexer = LexerBuilder.BuildLexer(new BuildResult>(), extensionBuilder, I18n, lexerPostProcess, implicitTokens); + return lexer; } @@ -200,7 +217,7 @@ public ParserBuilder() : this(null) var r = BuildNonTerminal(ntAndRule); r.SetVisitor(m); r.NonTerminalName = ntAndRule.Item1; - var key = ntAndRule.Item1 + "__" + r.Key; + var key = $"{ntAndRule.Item1}__{r.Key}"; functions[key] = m; NonTerminal nonT = null; if (!nonTerminals.ContainsKey(ntAndRule.Item1)) diff --git a/sly/parser/generator/ParserConfiguration.cs b/sly/parser/generator/ParserConfiguration.cs index 8cbde258..88608c8a 100644 --- a/sly/parser/generator/ParserConfiguration.cs +++ b/sly/parser/generator/ParserConfiguration.cs @@ -1,6 +1,8 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Text; +using sly.parser.syntax.grammar; namespace sly.parser.generator { @@ -16,6 +18,45 @@ public void AddNonTerminalIfNotExists(NonTerminal nonTerminal) if (!NonTerminals.ContainsKey(nonTerminal.Name)) NonTerminals[nonTerminal.Name] = nonTerminal; } + public bool HasImplicitTokens() => GetAllImplicitTokenClauses().Any(); + + public List> GetAllImplicitTokenClauses() + { + List> clauses = new List>(); + foreach (var nonTerminal in NonTerminals.Values) + { + foreach (var rule in nonTerminal.Rules) + { + foreach (var clause in rule.Clauses) + { + if (clause is TerminalClause terminalClause && terminalClause.IsImplicitToken) + { + clauses.Add(terminalClause); + } + + if (clause is ChoiceClause choices) + { + foreach (var choice in choices.Choices) + { + if (choice is TerminalClause terminal && terminal.IsImplicitToken) + { + clauses.Add(terminal); + } + } + } + + if (clause is OptionClause option) + { + if (option.Clause is TerminalClause terminal && terminal.IsImplicitToken) + clauses.Add(terminal); + } + } + } + } + + return clauses; + } + [ExcludeFromCodeCoverage] public string Dump() { diff --git a/sly/parser/generator/RuleParser.cs b/sly/parser/generator/RuleParser.cs index 572d2ae5..86f84b6a 100644 --- a/sly/parser/generator/RuleParser.cs +++ b/sly/parser/generator/RuleParser.cs @@ -88,6 +88,13 @@ public IClause ChoicesOne(Token head) return new ChoiceClause(choice); } + [Production("choices : STRING")] + public IClause ChoicesString(Token head) + { + var choice = BuildTerminalOrNonTerimal(head.Value, discard: false,implicitToken:true); + return new ChoiceClause(choice); + } + [Production("choices : IDENTIFIER OR choices ")] public IClause ChoicesMany(Token head, Token discardOr, ChoiceClause tail) { @@ -95,6 +102,13 @@ public IClause ChoicesMany(Token head, Token(headClause,tail.Choices); } + [Production("choices : STRING OR choices ")] + public IClause ChoicesManyImplicit(Token head, Token discardOr, ChoiceClause tail) + { + var headClause = BuildTerminalOrNonTerimal(head.Value,discard:false,implicitToken:true); + return new ChoiceClause(headClause,tail.Choices); + } + [Production("clause : IDENTIFIER ")] public IClause SimpleClause(Token id) @@ -103,6 +117,18 @@ public IClause SimpleClause(Token id) return clause; } + [Production("clause : STRING")] + public IClause ImplicitTokenClause(Token implicitToken) { + var clause = BuildTerminalOrNonTerimal(implicitToken.Value,discard:false, implicitToken :true); + return clause; + } + + [Production("clause : STRING DISCARD")] + public IClause ImplicitTokenClauseDiscarded(Token implicitToken, Token discard) { + var clause = BuildTerminalOrNonTerimal(implicitToken.Value,discard:true, implicitToken :true); + return clause; + } + #region groups @@ -192,8 +218,9 @@ public GroupClause GroupChoiceClause(ChoiceClause choices) #endregion - private IClause BuildTerminalOrNonTerimal(string name, bool discard = false) + private IClause BuildTerminalOrNonTerimal(string name, bool discard = false, bool implicitToken = false) { + var token = default(IN); IClause clause; var isTerminal = false; @@ -203,21 +230,30 @@ private IClause BuildTerminalOrNonTerimal(string name, bool discard = false) isTerminal = true; } + + if (isTerminal) clause = new TerminalClause(token, discard); else { - switch (name) + if (name.StartsWith("'")) + { + clause = new TerminalClause(name.Substring(1,name.Length-2), discard); + } + else { - case "INDENT": - clause = new IndentTerminalClause(IndentationType.Indent,discard); - break; - case "UINDENT": - clause = new IndentTerminalClause(IndentationType.UnIndent,discard); - break; - default: - clause = new NonTerminalClause(name); - break; + switch (name) + { + case "INDENT": + clause = new IndentTerminalClause(IndentationType.Indent, discard); + break; + case "UINDENT": + clause = new IndentTerminalClause(IndentationType.UnIndent, discard); + break; + default: + clause = new NonTerminalClause(name); + break; + } } } diff --git a/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs b/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs index 9f9a0775..ebaba2f9 100644 --- a/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs +++ b/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs @@ -35,7 +35,14 @@ private DotNode Leaf(SyntaxLeaf leaf) private DotNode Leaf(IN type, string value) { string label = type.ToString(); - label += "\n"; + if (label == "0") + { + label = ""; + } + else + { + label += "\n"; + } var esc = value.Replace("\"", "\\\""); label += "\\\"" + esc + "\\\""; var node = new DotNode(NodeCounter.ToString()) @@ -116,11 +123,6 @@ private DotNode Visit(OptionSyntaxNode node) private string GetNodeLabel(SyntaxNode node) { string label = node.Name; - if (node.IsExpressionNode) - { - label = node.Operation.OperatorToken.ToString(); - } - return label; } @@ -138,31 +140,22 @@ private DotNode Visit(SyntaxNode node) children.Add(v); } - if (node.IsByPassNode) - { - //result = children[0]; - } - else - { + - result = Node(GetNodeLabel(node)); - Graph.Add(result); - children.ForEach(c => + result = Node(GetNodeLabel(node)); + Graph.Add(result); + children.ForEach(c => + { + if (c != null) // Prevent arrows with null destinations { - if (c != null) // Prevent arrows with null destinations + var edge = new DotArrow(result, c) { - var edge = new DotArrow(result, c) - { - // Set all available properties - ArrowHeadShape = "none" - }; - Graph.Add(edge); - } - }); - - } - - + // Set all available properties + ArrowHeadShape = "none" + }; + Graph.Add(edge); + } + }); return result; } diff --git a/sly/parser/generator/visitor/dotgraph/DotArrow.cs b/sly/parser/generator/visitor/dotgraph/DotArrow.cs index 7ad9ee03..e1153373 100644 --- a/sly/parser/generator/visitor/dotgraph/DotArrow.cs +++ b/sly/parser/generator/visitor/dotgraph/DotArrow.cs @@ -4,13 +4,13 @@ namespace sly.parser.generator.visitor.dotgraph { public class DotArrow : IDot { - private DotNode source; - private DotNode destination; + public DotNode Source { get; private set; } + public DotNode Destination { get; private set; } public DotArrow(DotNode src, DotNode dest) { - source = src; - destination = dest; + Source = src; + Destination = dest; } public string Attribute(string name, string value) @@ -32,12 +32,17 @@ public string Attribute(string name, double value) public string ToGraph() { var builder = new StringBuilder(); - builder.Append($"{source.Name}->{destination?.Name} [ "); + builder.Append($"{Source.Name}->{Destination?.Name} [ "); builder.Append(Attribute("arrowshape", ArrowHeadShape)); builder.Append("];"); return builder.ToString(); } + + public string ToString() + { + return ToGraph(); + } } } \ No newline at end of file diff --git a/sly/parser/generator/visitor/dotgraph/DotGraph.cs b/sly/parser/generator/visitor/dotgraph/DotGraph.cs index 232aafda..80ea225d 100644 --- a/sly/parser/generator/visitor/dotgraph/DotGraph.cs +++ b/sly/parser/generator/visitor/dotgraph/DotGraph.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Text; namespace sly.parser.generator.visitor.dotgraph @@ -45,5 +46,47 @@ public string Compile() builder.AppendLine("}"); return builder.ToString(); } + + public List FindRoots() + { + var roots = edges.Where(x => !edges.Any(y => y.Destination?.Name == x.Source?.Name)); + return roots.Select(x => x.Source).ToList(); + } + + public IList FindEgdes(DotNode node) + { + var nodeEdges = edges.Where(x => x.Source?.Name == node.Name); + return nodeEdges.ToList(); + } + + public string Dump() + { + var roots = FindRoots(); + return string.Join("\n\n",roots.Select(x => Dump("",x))); + } + + private string Dump(string tab, DotNode node) + { + if (node == null) + { + return ""; + } + StringBuilder builder = new StringBuilder(); + builder.Append(tab) + .AppendLine(node.Label); + var edges = FindEgdes(node); + if (edges != null && edges.Any()) + { + foreach (var edge in edges) + { + builder.AppendLine(Dump(tab + "\t", edge.Destination)); + } + } + return builder.ToString(); + } + + + + } } \ No newline at end of file diff --git a/sly/parser/parser/GroupItem.cs b/sly/parser/parser/GroupItem.cs index 1c4f2d11..8588d2dc 100644 --- a/sly/parser/parser/GroupItem.cs +++ b/sly/parser/parser/GroupItem.cs @@ -39,11 +39,13 @@ public X Match(Func, X> fToken, Func fValue return fValue(Name, Value); } + [ExcludeFromCodeCoverage] public static implicit operator OUT(GroupItem item) { return item.Match((name, token) => default(OUT), (name, value) => item.Value); } + [ExcludeFromCodeCoverage] public static implicit operator Token(GroupItem item) { return item.Match>((name, token) => item.Token, (name, value) => default(Token)); diff --git a/sly/parser/parser/ISyntaxParser.cs b/sly/parser/parser/ISyntaxParser.cs index f13bfc37..7aa6f088 100644 --- a/sly/parser/parser/ISyntaxParser.cs +++ b/sly/parser/parser/ISyntaxParser.cs @@ -11,5 +11,7 @@ public interface ISyntaxParser where IN : struct SyntaxParseResult Parse(IList> tokens, string startingNonTerminal = null); void Init(ParserConfiguration configuration, string root); + + string Dump(); } } \ No newline at end of file diff --git a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs index d948b0d6..fdecc415 100644 --- a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs +++ b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs @@ -260,15 +260,7 @@ public EBNFRecursiveDescentSyntaxParser(ParserConfiguration configurati return finalResult; } } - - } - else - { - throw new ParserConfigurationException( - $@"expression rule {rule.RuleString} is incorrect : must have ""nonterminal terminal nonterminal"" scheme"); } - - } var result = new SyntaxParseResult(); diff --git a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParserStarter.cs b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParserStarter.cs index 7ec39f68..c365a8dd 100644 --- a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParserStarter.cs +++ b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParserStarter.cs @@ -15,19 +15,6 @@ public partial class EBNFRecursiveDescentSyntaxParser where IN : struct { switch (first) { - case TerminalClause clause: - { - var term = clause; - - InitStartingTokensWithTerminal(rule, term); - break; - } - case NonTerminalClause clause: - { - var nonterm = clause; - InitStartingTokensWithNonTerminal(rule, nonterm, nonTerminals); - break; - } case ZeroOrMoreClause zeroOrMore: { InitStartingTokensWithZeroOrMore(rule, zeroOrMore, nonTerminals); @@ -44,10 +31,10 @@ public partial class EBNFRecursiveDescentSyntaxParser where IN : struct rule.PossibleLeadingTokens.Add(terminalClause.ExpectedToken); break; } - case NonTerminalClause terminalClause: + case NonTerminalClause nonTerminalClause: { - InitStartingTokensForNonTerminal(nonTerminals, terminalClause.NonTerminalName); - NonTerminal nonTerminal = nonTerminals[terminalClause.NonTerminalName]; + InitStartingTokensForNonTerminal(nonTerminals, nonTerminalClause.NonTerminalName); + NonTerminal nonTerminal = nonTerminals[nonTerminalClause.NonTerminalName]; { rule.PossibleLeadingTokens.AddRange(nonTerminal.PossibleLeadingTokens); } diff --git a/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs b/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs index d2253bf4..f96b625d 100644 --- a/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs +++ b/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs @@ -249,8 +249,11 @@ public SyntaxParseResult ParseTerminal(IList> tokens, TerminalClau token.Discarded = terminal.Discarded; result.Root = new SyntaxLeaf(token, terminal.Discarded); result.HasByPassNodes = false; - result.Errors.Add(new UnexpectedTokenSyntaxError(token,I18n,terminal.ExpectedToken)); - result.AddExpecting(terminal.ExpectedToken); + if (result.IsError) + { + result.Errors.Add(new UnexpectedTokenSyntaxError(token, I18n, terminal.ExpectedToken)); + result.AddExpecting(terminal.ExpectedToken); + } return result; } @@ -415,6 +418,8 @@ public virtual void Init(ParserConfiguration configuration, string root InitializeStartingTokens(configuration, StartingNonTerminal); } + + #endregion } } \ No newline at end of file diff --git a/sly/parser/parser/llparser/RecursiveDescentSyntaxParserStarter.cs b/sly/parser/parser/llparser/RecursiveDescentSyntaxParserStarter.cs index f4286000..a06e48b9 100644 --- a/sly/parser/parser/llparser/RecursiveDescentSyntaxParserStarter.cs +++ b/sly/parser/parser/llparser/RecursiveDescentSyntaxParserStarter.cs @@ -165,5 +165,10 @@ protected virtual void InitializeStartingTokens(ParserConfiguration con #endregion + public string Dump() + { + return this.Configuration.Dump(); + } + } } \ No newline at end of file diff --git a/sly/parser/syntax/grammar/ChoiceClause.cs b/sly/parser/syntax/grammar/ChoiceClause.cs index 1431ddeb..7d606e44 100644 --- a/sly/parser/syntax/grammar/ChoiceClause.cs +++ b/sly/parser/syntax/grammar/ChoiceClause.cs @@ -31,7 +31,8 @@ public ChoiceClause(IClause choice, List> choices) : this(choice) [ExcludeFromCodeCoverage] public override string ToString() { - return string.Join(" | ", Choices.Select, string>(c => c.ToString())); + var choices = string.Join(" | ", Choices.Select, string>(c => c.Dump())); + return $"[ {choices} ]"; } public bool MayBeEmpty() diff --git a/sly/parser/syntax/grammar/NonTerminalClause.cs b/sly/parser/syntax/grammar/NonTerminalClause.cs index 92eb5136..85fb9ac5 100644 --- a/sly/parser/syntax/grammar/NonTerminalClause.cs +++ b/sly/parser/syntax/grammar/NonTerminalClause.cs @@ -28,7 +28,7 @@ public override string ToString() [ExcludeFromCodeCoverage] public string Dump() { - return NonTerminalName; + return $"{NonTerminalName}(NT)"; } } } \ No newline at end of file diff --git a/sly/parser/syntax/grammar/TerminalClause.cs b/sly/parser/syntax/grammar/TerminalClause.cs index 98b82677..b74b2148 100644 --- a/sly/parser/syntax/grammar/TerminalClause.cs +++ b/sly/parser/syntax/grammar/TerminalClause.cs @@ -15,9 +15,23 @@ public TerminalClause(T token, bool discard) : this(token) { Discarded = discard; } + + public TerminalClause(string implicitToken, bool discard) : this(default(T)) + { + ImplicitToken = implicitToken; + Discarded = discard; + } + + public TerminalClause(string implicitToken) : this(implicitToken,false) + { + } public T ExpectedToken { get; set; } + public string ImplicitToken { get; set; } + + public bool IsImplicitToken => !string.IsNullOrEmpty(ImplicitToken); + public bool Discarded { get; set; } public virtual bool MayBeEmpty() @@ -27,6 +41,10 @@ public virtual bool MayBeEmpty() public virtual bool Check(Token nextToken) { + if (IsImplicitToken) + { + return nextToken.Value.Equals(ImplicitToken); + } return nextToken.TokenID.Equals(ExpectedToken); } @@ -34,14 +52,28 @@ public virtual bool Check(Token nextToken) public override string ToString() { var b = new StringBuilder(); - b.Append(ExpectedToken); + if (IsImplicitToken) + { + b.Append($"'{ImplicitToken}'"); + } + else + { + b.Append(ExpectedToken); + } + if (Discarded) b.Append("[d]"); + b.Append("(T)"); return b.ToString(); } public virtual string Dump() { - return ExpectedToken.ToString(); + + if (IsImplicitToken) + { + return $"'{ImplicitToken}'(T)"; + } + return $"{ExpectedToken}(T)"; } } diff --git a/sly/parser/syntax/grammar/ZeroOrMoreClause.cs b/sly/parser/syntax/grammar/ZeroOrMoreClause.cs index f6829819..c97d2818 100644 --- a/sly/parser/syntax/grammar/ZeroOrMoreClause.cs +++ b/sly/parser/syntax/grammar/ZeroOrMoreClause.cs @@ -24,7 +24,8 @@ public override bool MayBeEmpty() [ExcludeFromCodeCoverage] public override string Dump() { - return Clause.Dump()+"*"; + var t = Clause.Dump() + "*"; + return t; } } } \ No newline at end of file diff --git a/sly/parser/syntax/tree/ISyntaxNode.cs b/sly/parser/syntax/tree/ISyntaxNode.cs index cb21cc95..cf08d93c 100644 --- a/sly/parser/syntax/tree/ISyntaxNode.cs +++ b/sly/parser/syntax/tree/ISyntaxNode.cs @@ -9,5 +9,8 @@ public interface ISyntaxNode where IN : struct bool HasByPassNodes { get; set; } string Dump(string tab); + + string ToJson(int index = 0); + } } \ No newline at end of file diff --git a/sly/parser/syntax/tree/SyntaxEpsilon.cs b/sly/parser/syntax/tree/SyntaxEpsilon.cs index 875adb8c..cdabf91a 100644 --- a/sly/parser/syntax/tree/SyntaxEpsilon.cs +++ b/sly/parser/syntax/tree/SyntaxEpsilon.cs @@ -1,23 +1,25 @@ using sly.lexer; +using System.Diagnostics.CodeAnalysis; namespace sly.parser.syntax.tree { public class SyntaxEpsilon : ISyntaxNode where IN : struct { - public SyntaxEpsilon() - { - - } - - public bool Discarded { get; } = false; public string Name => "Epsilon"; public bool HasByPassNodes { get; set; } = false; + [ExcludeFromCodeCoverage] public string Dump(string tab) { return $"Epsilon"; } + + [ExcludeFromCodeCoverage] + public string ToJson(int index = 0) + { + return $@"""{index}.Epsilon"":""e"""; + } } -} \ No newline at end of file +} diff --git a/sly/parser/syntax/tree/SyntaxLeaf.cs b/sly/parser/syntax/tree/SyntaxLeaf.cs index 7e31be8f..dabde88d 100644 --- a/sly/parser/syntax/tree/SyntaxLeaf.cs +++ b/sly/parser/syntax/tree/SyntaxLeaf.cs @@ -20,7 +20,14 @@ public SyntaxLeaf(Token token, bool discarded) public string Dump(string tab) { - return $"{tab}+ {Token.TokenID} : {Token.Value} @{Token.PositionInTokenFlow}"; + return $"{tab}+ {Token.TokenID.ToString()} : {Token.Value} @{Token.PositionInTokenFlow}"; } + + public string ToJson(int index = 0) + { + return $@"""{index}.{Token.TokenID.ToString()}"" : ""{Token.Value}"""; + } + + } } \ No newline at end of file diff --git a/sly/parser/syntax/tree/SyntaxNode.cs b/sly/parser/syntax/tree/SyntaxNode.cs index efc737b4..bfa2527e 100644 --- a/sly/parser/syntax/tree/SyntaxNode.cs +++ b/sly/parser/syntax/tree/SyntaxNode.cs @@ -83,7 +83,23 @@ public ISyntaxNode Right public string Dump(string tab) { StringBuilder builder = new StringBuilder(); + string expressionSuffix = ""; + if (Operation != null && (Operation.IsBinary || Operation.IsBinary)) + { + if (Operation.IsImplicitOperatorToken) + { + expressionSuffix = Operation.ImplicitOperatorToken; + } + else + { + expressionSuffix = Operation.OperatorToken.ToString(); + } + + expressionSuffix = $">{expressionSuffix}<"; + } + builder.AppendLine($"{tab}+ {Name} {(IsByPassNode ? "===":"")}"); + foreach (var child in Children) { builder.AppendLine($"{child.Dump(tab + "\t")}"); @@ -91,6 +107,36 @@ public string Dump(string tab) return builder.ToString(); } + + public string ToJson(int index = 0) + { + StringBuilder builder = new StringBuilder(); + + + builder.Append($@"""{index}.{Name}"); + if (IsByPassNode) + { + builder.Append("--"); + } + + builder.AppendLine(@""" : {"); + + for (int i = 0; i < Children.Count; i++) + { + var child = Children[i]; + builder.Append(child.ToJson(i)); + if (i < Children.Count - 1) + { + builder.Append(","); + } + + builder.AppendLine(); + } + + builder.Append("}"); + + return builder.ToString(); + } #endregion