From 9bc728cd18b0491ac9719d51a9784ccae91dee88 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 31 May 2022 15:04:59 +0200 Subject: [PATCH 01/22] implicit tokens : initial --- sly/parser/generator/EbnfToken.cs | 5 ++++- sly/parser/generator/RuleParser.cs | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/sly/parser/generator/EbnfToken.cs b/sly/parser/generator/EbnfToken.cs index 496ed4ac..aba98ca0 100644 --- a/sly/parser/generator/EbnfToken.cs +++ b/sly/parser/generator/EbnfToken.cs @@ -52,7 +52,10 @@ public enum EbnfTokenGeneric LCROG = 11, [Lexeme(GenericToken.SugarToken,"]")] - RCROG = 12 + RCROG = 12, + + [Lexeme(GenericToken.String, "'","")] + STRING = 13, } } \ No newline at end of file diff --git a/sly/parser/generator/RuleParser.cs b/sly/parser/generator/RuleParser.cs index 572d2ae5..d9c3f09f 100644 --- a/sly/parser/generator/RuleParser.cs +++ b/sly/parser/generator/RuleParser.cs @@ -103,6 +103,12 @@ public IClause SimpleClause(Token id) return clause; } + [Production("clause : STRING DISCARD?")] + public IClause ImplicitTokenClause(Token implicitToken, Token discard) { + var clause = BuildTerminalOrNonTerimal(implicitToken.StringWithoutQuotes,discard:!discard.IsEmpty, implicitToken :true); + return clause; + } + #region groups @@ -192,7 +198,7 @@ public GroupClause GroupChoiceClause(ChoiceClause choices) #endregion - private IClause BuildTerminalOrNonTerimal(string name, bool discard = false) + private IClause BuildTerminalOrNonTerimal(string name, bool discard = false, bool implicitToken = false) { var token = default(IN); IClause clause; From 15728fed16823718c7ed0c40063ddb3bece9f7c8 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 31 May 2022 15:19:39 +0200 Subject: [PATCH 02/22] fix grammar rule parsing --- sly/lexer/Token.cs | 2 ++ sly/parser/generator/EbnfToken.cs | 2 +- sly/parser/generator/RuleParser.cs | 26 +++++++++++++++++++++++--- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/sly/lexer/Token.cs b/sly/lexer/Token.cs index 41cebe6b..7372c694 100644 --- a/sly/lexer/Token.cs +++ b/sly/lexer/Token.cs @@ -113,6 +113,8 @@ public Token Previous(int channelId) public bool IsWhiteSpace { get; set; } public bool IsEOL { get; set; } + + public bool IsImplicit { get; set; } public CommentType CommentType { get; set; } = CommentType.No; diff --git a/sly/parser/generator/EbnfToken.cs b/sly/parser/generator/EbnfToken.cs index aba98ca0..94fb6174 100644 --- a/sly/parser/generator/EbnfToken.cs +++ b/sly/parser/generator/EbnfToken.cs @@ -54,7 +54,7 @@ public enum EbnfTokenGeneric [Lexeme(GenericToken.SugarToken,"]")] RCROG = 12, - [Lexeme(GenericToken.String, "'","")] + [Lexeme(GenericToken.String, "'","\\")] STRING = 13, } diff --git a/sly/parser/generator/RuleParser.cs b/sly/parser/generator/RuleParser.cs index d9c3f09f..fc281826 100644 --- a/sly/parser/generator/RuleParser.cs +++ b/sly/parser/generator/RuleParser.cs @@ -88,6 +88,13 @@ public IClause ChoicesOne(Token head) return new ChoiceClause(choice); } + [Production("choices : STRING")] + public IClause ChoicesString(Token head) + { + var choice = BuildTerminalOrNonTerimal(head.StringWithoutQuotes, discard: false,implicitToken:true); + return new ChoiceClause(choice); + } + [Production("choices : IDENTIFIER OR choices ")] public IClause ChoicesMany(Token head, Token discardOr, ChoiceClause tail) { @@ -95,6 +102,13 @@ public IClause ChoicesMany(Token head, Token(headClause,tail.Choices); } + [Production("choices : STRING OR choices ")] + public IClause ChoicesManyImplicit(Token head, Token discardOr, ChoiceClause tail) + { + var headClause = BuildTerminalOrNonTerimal(head.StringWithoutQuotes,discard:false,implicitToken:true); + return new ChoiceClause(headClause,tail.Choices); + } + [Production("clause : IDENTIFIER ")] public IClause SimpleClause(Token id) @@ -103,9 +117,15 @@ public IClause SimpleClause(Token id) return clause; } - [Production("clause : STRING DISCARD?")] - public IClause ImplicitTokenClause(Token implicitToken, Token discard) { - var clause = BuildTerminalOrNonTerimal(implicitToken.StringWithoutQuotes,discard:!discard.IsEmpty, implicitToken :true); + [Production("clause : STRING")] + public IClause ImplicitTokenClause(Token implicitToken) { + var clause = BuildTerminalOrNonTerimal(implicitToken.StringWithoutQuotes,discard:false, implicitToken :true); + return clause; + } + + [Production("clause : STRING DISCARD")] + public IClause ImplicitTokenClauseDiscarded(Token implicitToken, Token discard) { + var clause = BuildTerminalOrNonTerimal(implicitToken.StringWithoutQuotes,discard:true, implicitToken :true); return clause; } From b1f1deb0d6f3680e46b6682e280de92f0e37b548 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 31 May 2022 16:15:11 +0200 Subject: [PATCH 03/22] implicit tokens : working ebnf grammar rule parser --- ParserTests/ImplicitTokensTests.cs | 144 ++++++++++++++++++++ sly/parser/generator/EBNFParserBuilder.cs | 13 +- sly/parser/generator/ParserBuilder.cs | 16 ++- sly/parser/generator/ParserConfiguration.cs | 35 +++++ sly/parser/generator/RuleParser.cs | 38 ++++-- sly/parser/syntax/grammar/TerminalClause.cs | 10 ++ 6 files changed, 235 insertions(+), 21 deletions(-) create mode 100644 ParserTests/ImplicitTokensTests.cs diff --git a/ParserTests/ImplicitTokensTests.cs b/ParserTests/ImplicitTokensTests.cs new file mode 100644 index 00000000..fb5490c3 --- /dev/null +++ b/ParserTests/ImplicitTokensTests.cs @@ -0,0 +1,144 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using expressionparser; +using indented; +using jsonparser; +using jsonparser.JsonModel; +using simpleExpressionParser; +using sly.buildresult; +using sly.lexer; +using sly.parser; +using sly.parser.generator; +using sly.parser.generator.visitor; +using sly.parser.llparser; +using sly.parser.parser; +using sly.parser.syntax.grammar; +using Xunit; +using String = System.String; + +namespace ParserTests +{ + + [Lexer(IgnoreWS = true, IgnoreEOL = true)] + public enum ImplicitTokensTokens + { + [MultiLineComment("/*","*/")] + MULTILINECOMMENT = 1, + + [SingleLineComment("//")] + SINGLELINECOMMENT = 2, + + [Lexeme(GenericToken.Identifier, IdentifierType.AlphaNumeric)] + ID = 3, + + [Lexeme(GenericToken.Double, channel:101)] + DOUBLE = 4 + } + + public class ImplicitTokensParser + { + [Production("primary: DOUBLE")] + public double Primary(Token doubleToken) + { + return doubleToken.DoubleValue; + } + + + [Production("expression : term ['+' | '-'] expression")] + + public double Expression(double left, Token operatorToken, double right) + { + double result = 0.0; + + + switch (operatorToken.StringWithoutQuotes) + { + case "+": + { + result = left + right; + break; + } + case "-": + { + result = left - right; + break; + } + } + + return result; + } + + [Production("expression : term")] + public double Expression_Term(double termValue) + { + return termValue; + } + + [Production("term : factor ['*' | '/'] term")] + public double Term(double left, Token operatorToken, double right) + { + var result = 0d; + + + switch (operatorToken.StringWithoutQuotes) + { + case "*": + { + result = left * right; + break; + } + case "/": + { + result = left / right; + break; + } + } + + return result; + } + + [Production("term : factor")] + public double Term_Factor(double factorValue) + { + return factorValue; + } + + [Production("factor : primary")] + public double primaryFactor(double primValue) + { + return primValue; + } + + [Production("factor : '-'[d] factor")] + public double Factor(double factorValue) + { + return -factorValue; + } + } + + + public class ImplicitTokensTests + { + private Parser Parser; + + private BuildResult> BuildParser() + { + var parserInstance = new ImplicitTokensParser(); + var builder = new ParserBuilder(); + var result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "expression"); + return result; + } + + [Fact] + public void BuildParserTest() + { + var parser = BuildParser(); + Assert.True(parser.IsOk); + Assert.NotNull(parser.Result); + } + } + +} diff --git a/sly/parser/generator/EBNFParserBuilder.cs b/sly/parser/generator/EBNFParserBuilder.cs index 986c7d92..29b5719f 100644 --- a/sly/parser/generator/EBNFParserBuilder.cs +++ b/sly/parser/generator/EBNFParserBuilder.cs @@ -38,6 +38,8 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) { configuration = ExtractEbnfParserConfiguration(parserInstance.GetType(), grammarParser); LeftRecursionChecker recursionChecker = new LeftRecursionChecker(); + + // check left recursion. var (foundRecursion, recursions) = LeftRecursionChecker.CheckLeftRecursion(configuration); if (foundRecursion) { @@ -46,7 +48,6 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) I18N.Instance.GetText(I18n,Message.LeftRecursion,recs), ErrorCodes.PARSER_LEFT_RECURSIVE)); return result; - } configuration.StartingRule = rootRule; @@ -71,7 +72,7 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) } var parser = new Parser(I18n,syntaxParser, visitor); parser.Configuration = configuration; - var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess); + var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess, configuration); if (lexerResult.IsError) { foreach (var lexerResultError in lexerResult.Errors) @@ -81,10 +82,12 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) return result; } else + { parser.Lexer = lexerResult.Result; - parser.Instance = parserInstance; - result.Result = parser; - return result; + parser.Instance = parserInstance; + result.Result = parser; + return result; + } } diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index 61735ce1..2a47f27a 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -167,9 +167,21 @@ public ParserBuilder() : this(null) protected virtual BuildResult> BuildLexer(BuildExtension extensionBuilder = null, - LexerPostProcess lexerPostProcess = null) + LexerPostProcess lexerPostProcess = null, ParserConfiguration parserConfiguration = null) { var lexer = LexerBuilder.BuildLexer(new BuildResult>(), extensionBuilder, I18n, lexerPostProcess); + if (parserConfiguration != null) + { + var implicitTokenClauses = parserConfiguration.GetAllImplicitTokenClauses(); + if (implicitTokenClauses.Any()) + { + Console.WriteLine($"found {implicitTokenClauses.Count} implicit tokens"); + var implicits = implicitTokenClauses.Select(x => x.ImplicitToken).Distinct(); + Console.WriteLine(string.Join(", ", implicits)); + ; // TODO + } + } + return lexer; } @@ -200,7 +212,7 @@ public ParserBuilder() : this(null) var r = BuildNonTerminal(ntAndRule); r.SetVisitor(m); r.NonTerminalName = ntAndRule.Item1; - var key = ntAndRule.Item1 + "__" + r.Key; + var key = $"{ntAndRule.Item1}__{r.Key}"; functions[key] = m; NonTerminal nonT = null; if (!nonTerminals.ContainsKey(ntAndRule.Item1)) diff --git a/sly/parser/generator/ParserConfiguration.cs b/sly/parser/generator/ParserConfiguration.cs index 8cbde258..45179881 100644 --- a/sly/parser/generator/ParserConfiguration.cs +++ b/sly/parser/generator/ParserConfiguration.cs @@ -1,6 +1,8 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Text; +using sly.parser.syntax.grammar; namespace sly.parser.generator { @@ -16,6 +18,39 @@ public void AddNonTerminalIfNotExists(NonTerminal nonTerminal) if (!NonTerminals.ContainsKey(nonTerminal.Name)) NonTerminals[nonTerminal.Name] = nonTerminal; } + public bool HasImplicitTokens() => GetAllImplicitTokenClauses().Any(); + + public List> GetAllImplicitTokenClauses() + { + List> clauses = new List>(); + foreach (var nonTerminal in NonTerminals.Values) + { + foreach (var rule in nonTerminal.Rules) + { + foreach (var clause in rule.Clauses) + { + if (clause is TerminalClause terminalClause && terminalClause.IsImplicitToken) + { + clauses.Add(terminalClause); + } + + if (clause is ChoiceClause choices) + { + foreach (var choice in choices.Choices) + { + if (choice is TerminalClause terminal && terminal.IsImplicitToken) + { + clauses.Add(terminal); + } + } + } + } + } + } + + return clauses; + } + [ExcludeFromCodeCoverage] public string Dump() { diff --git a/sly/parser/generator/RuleParser.cs b/sly/parser/generator/RuleParser.cs index fc281826..86f84b6a 100644 --- a/sly/parser/generator/RuleParser.cs +++ b/sly/parser/generator/RuleParser.cs @@ -91,7 +91,7 @@ public IClause ChoicesOne(Token head) [Production("choices : STRING")] public IClause ChoicesString(Token head) { - var choice = BuildTerminalOrNonTerimal(head.StringWithoutQuotes, discard: false,implicitToken:true); + var choice = BuildTerminalOrNonTerimal(head.Value, discard: false,implicitToken:true); return new ChoiceClause(choice); } @@ -105,7 +105,7 @@ public IClause ChoicesMany(Token head, Token ChoicesManyImplicit(Token head, Token discardOr, ChoiceClause tail) { - var headClause = BuildTerminalOrNonTerimal(head.StringWithoutQuotes,discard:false,implicitToken:true); + var headClause = BuildTerminalOrNonTerimal(head.Value,discard:false,implicitToken:true); return new ChoiceClause(headClause,tail.Choices); } @@ -119,13 +119,13 @@ public IClause SimpleClause(Token id) [Production("clause : STRING")] public IClause ImplicitTokenClause(Token implicitToken) { - var clause = BuildTerminalOrNonTerimal(implicitToken.StringWithoutQuotes,discard:false, implicitToken :true); + var clause = BuildTerminalOrNonTerimal(implicitToken.Value,discard:false, implicitToken :true); return clause; } [Production("clause : STRING DISCARD")] public IClause ImplicitTokenClauseDiscarded(Token implicitToken, Token discard) { - var clause = BuildTerminalOrNonTerimal(implicitToken.StringWithoutQuotes,discard:true, implicitToken :true); + var clause = BuildTerminalOrNonTerimal(implicitToken.Value,discard:true, implicitToken :true); return clause; } @@ -220,6 +220,7 @@ public GroupClause GroupChoiceClause(ChoiceClause choices) private IClause BuildTerminalOrNonTerimal(string name, bool discard = false, bool implicitToken = false) { + var token = default(IN); IClause clause; var isTerminal = false; @@ -229,21 +230,30 @@ private IClause BuildTerminalOrNonTerimal(string name, bool discard = false, isTerminal = true; } + + if (isTerminal) clause = new TerminalClause(token, discard); else { - switch (name) + if (name.StartsWith("'")) { - case "INDENT": - clause = new IndentTerminalClause(IndentationType.Indent,discard); - break; - case "UINDENT": - clause = new IndentTerminalClause(IndentationType.UnIndent,discard); - break; - default: - clause = new NonTerminalClause(name); - break; + clause = new TerminalClause(name.Substring(1,name.Length-2), discard); + } + else + { + switch (name) + { + case "INDENT": + clause = new IndentTerminalClause(IndentationType.Indent, discard); + break; + case "UINDENT": + clause = new IndentTerminalClause(IndentationType.UnIndent, discard); + break; + default: + clause = new NonTerminalClause(name); + break; + } } } diff --git a/sly/parser/syntax/grammar/TerminalClause.cs b/sly/parser/syntax/grammar/TerminalClause.cs index 98b82677..a4fc7886 100644 --- a/sly/parser/syntax/grammar/TerminalClause.cs +++ b/sly/parser/syntax/grammar/TerminalClause.cs @@ -15,9 +15,19 @@ public TerminalClause(T token, bool discard) : this(token) { Discarded = discard; } + + public TerminalClause(string implicitToken, bool discard) : this(default(T)) + { + ImplicitToken = implicitToken; + Discarded = discard; + } public T ExpectedToken { get; set; } + public string ImplicitToken { get; set; } + + public bool IsImplicitToken => !string.IsNullOrEmpty(ImplicitToken); + public bool Discarded { get; set; } public virtual bool MayBeEmpty() From 1ae320bf5431d080575eb5d8796588c0fce1faf9 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 31 May 2022 16:57:17 +0200 Subject: [PATCH 04/22] implicit tokens : adding implicit keywords and sugar, WIP --- ParserTests/ImplicitTokensTests.cs | 21 ++++++++++++++++++++- sly/lexer/fsm/FSMLexerBuilder.cs | 2 +- sly/parser/generator/ParserBuilder.cs | 27 +++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/ParserTests/ImplicitTokensTests.cs b/ParserTests/ImplicitTokensTests.cs index fb5490c3..008c1d68 100644 --- a/ParserTests/ImplicitTokensTests.cs +++ b/ParserTests/ImplicitTokensTests.cs @@ -35,7 +35,11 @@ public enum ImplicitTokensTokens ID = 3, [Lexeme(GenericToken.Double, channel:101)] - DOUBLE = 4 + DOUBLE = 4, + + [Keyword("Test")] + TEST = 5 + } public class ImplicitTokensParser @@ -117,6 +121,18 @@ public double Factor(double factorValue) { return -factorValue; } + + [Production("primary : 'bozzo'[d]")] + public double Bozzo() + { + return 42.0; + } + + [Production("primary : TEST[d]")] + public double Test() + { + return 0.0; + } } @@ -138,6 +154,9 @@ public void BuildParserTest() var parser = BuildParser(); Assert.True(parser.IsOk); Assert.NotNull(parser.Result); + var r = parser.Result.Parse("1.0 + 2.0 + bozzo"); + Assert.True(r.IsOk); + Assert.Equal(45.0,r.Result); } } diff --git a/sly/lexer/fsm/FSMLexerBuilder.cs b/sly/lexer/fsm/FSMLexerBuilder.cs index 5e95cac9..0d133354 100644 --- a/sly/lexer/fsm/FSMLexerBuilder.cs +++ b/sly/lexer/fsm/FSMLexerBuilder.cs @@ -13,7 +13,7 @@ public class FSMLexerBuilder { private int CurrentState; - private readonly Dictionary Marks; + internal readonly Dictionary Marks; public FSMLexerBuilder() diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index 2a47f27a..acf628ae 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -178,6 +178,33 @@ public ParserBuilder() : this(null) Console.WriteLine($"found {implicitTokenClauses.Count} implicit tokens"); var implicits = implicitTokenClauses.Select(x => x.ImplicitToken).Distinct(); Console.WriteLine(string.Join(", ", implicits)); + if (lexer.IsOk && lexer.Result is GenericLexer genericLexer) + { + Console.WriteLine("something really magic will happen here !"); + foreach (var @implicit in implicits) + { + var x = genericLexer.FSMBuilder.Fsm.Run(@implicit, new LexerPosition()); + if (x.IsSuccess) + { + // todo : if x.NodeId == "in_identifier" + var t = genericLexer.FSMBuilder.Marks; + var y = genericLexer.FSMBuilder.Marks.FirstOrDefault(k => k.Value == x.NodeId); + if (y.Key == GenericLexer.in_identifier) // implicit keyword + { + var result = new BuildResult>(); + genericLexer.AddKeyWord(default(IN),@implicit,result); + ; + } + Console.WriteLine("hey ! Rodriguez !"); + } + else + { + var result = new BuildResult>(); + genericLexer.AddSugarLexem(default(IN),result,@implicit); + } + } + ; + } ; // TODO } } From f28c8335116d3f474e565b4cd69bb7a3fc533917 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 31 May 2022 17:21:10 +0200 Subject: [PATCH 05/22] implicit tokens : first passing unit test ! --- ParserTests/ImplicitTokensTests.cs | 69 ++++++++---------------------- 1 file changed, 17 insertions(+), 52 deletions(-) diff --git a/ParserTests/ImplicitTokensTests.cs b/ParserTests/ImplicitTokensTests.cs index 008c1d68..96851d42 100644 --- a/ParserTests/ImplicitTokensTests.cs +++ b/ParserTests/ImplicitTokensTests.cs @@ -49,9 +49,22 @@ public double Primary(Token doubleToken) { return doubleToken.DoubleValue; } + + [Production("primary : 'bozzo'[d]")] + public double Bozzo() + { + return 42.0; + } + + [Production("primary : TEST[d]")] + public double Test() + { + return 0.0; + } - [Production("expression : term ['+' | '-'] expression")] + [Production("expression : primary ['+' | '-'] expression")] + public double Expression(double left, Token operatorToken, double right) { @@ -75,64 +88,16 @@ public double Expression(double left, Token operatorToken, return result; } - [Production("expression : term")] - public double Expression_Term(double termValue) - { - return termValue; - } - [Production("term : factor ['*' | '/'] term")] - public double Term(double left, Token operatorToken, double right) + [Production("expression : primary ")] + public double Simple(double value) { - var result = 0d; - - - switch (operatorToken.StringWithoutQuotes) - { - case "*": - { - result = left * right; - break; - } - case "/": - { - result = left / right; - break; - } - } - - return result; + return value; } - [Production("term : factor")] - public double Term_Factor(double factorValue) - { - return factorValue; - } - [Production("factor : primary")] - public double primaryFactor(double primValue) - { - return primValue; - } - - [Production("factor : '-'[d] factor")] - public double Factor(double factorValue) - { - return -factorValue; - } - [Production("primary : 'bozzo'[d]")] - public double Bozzo() - { - return 42.0; - } - [Production("primary : TEST[d]")] - public double Test() - { - return 0.0; - } } From 0764b616f49ec32d0f20d2904c7b1038581c0eb0 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 1 Jun 2022 07:48:37 +0200 Subject: [PATCH 06/22] implicit tokens : unit test --- .github/workflows/.dotnetcore.yml.un~ | Bin 0 -> 537 bytes .github/workflows/dotnetcore.yml | 1 + .github/workflows/dotnetcore.yml~ | 103 ++++++++++++++++++++++++++ ParserTests/ImplicitTokensTests.cs | 6 +- 4 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/.dotnetcore.yml.un~ create mode 100644 .github/workflows/dotnetcore.yml~ diff --git a/.github/workflows/.dotnetcore.yml.un~ b/.github/workflows/.dotnetcore.yml.un~ new file mode 100644 index 0000000000000000000000000000000000000000..35f1fb2567684aa8bab5acbe3f1a8af7bb4bc5de GIT binary patch literal 537 zcmWH`%$*;a=aT=Ffl1)vf~M}<>^~dzf>|yq9}qP+D>pOfVw38qta!M`&cd33fguBk zVSo?FV1#0j7>o^~8Iq>`-p2wFiN636Km#Bt4j={@!tfsmz}i8aQ9y$b07l1NXmos) g0GiJZ#9Tm}4#c3S;8OqtU4`7l;*!)N)5hnk0FE^yy8r+H literal 0 HcmV?d00001 diff --git a/.github/workflows/dotnetcore.yml b/.github/workflows/dotnetcore.yml index ec120cfc..d0777c13 100644 --- a/.github/workflows/dotnetcore.yml +++ b/.github/workflows/dotnetcore.yml @@ -6,6 +6,7 @@ on: - dev - tech/actions - master + - feature/implicit-tokens pull_request: branches: - dev diff --git a/.github/workflows/dotnetcore.yml~ b/.github/workflows/dotnetcore.yml~ new file mode 100644 index 00000000..ec120cfc --- /dev/null +++ b/.github/workflows/dotnetcore.yml~ @@ -0,0 +1,103 @@ +name: .NET Core + +on: + push: + branches: + - dev + - tech/actions + - master + pull_request: + branches: + - dev +permissions: + pull-requests: write +jobs: + build: + env: + TESTS_PROJECT: 'ParserTests/ParserTests.csproj' # path to test project or solution + PUBLISH_NUGET: true # if true a nuget will be published on version change + RUN_TESTS: true # if true tests are run and coverage data is published to coveralls and a coverage report is produced. + MAIN_CSPROJ: 'sly/sly.csproj' # main project (for nuget packaging) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - uses: actions/checkout@v2 + - name: Setup .NET Core + uses: actions/setup-dotnet@v1 + with: + dotnet-version: 3.1.101 + - name: Clean artifacts and nugets + run: dotnet clean --configuration Release && dotnet nuget locals all --clear + - name: Build with dotnet + run: dotnet build --configuration Release + - name: Test with dotnet + uses: b3b00/coverlet-action@1.2.2 + id: 'coverlet' + if: env.RUN_TESTS + with: + testProject: ${{env.TESTS_PROJECT}} + output: 'lcov.info' + threshold: 80 + outputFormat: 'lcov' + excludes: '[program]*,[expressionParser]*,[jsonparser]*,[while]*,[indentedWhile]*,[SimpleExpressionParser]*,[GenericLexerWithCallbacks]*,[indented]*' + - name: coveralls + uses: coverallsapp/github-action@v1.1.1 + if: matrix.os == 'windows-latest' && env.RUN_TESTS + with: + github-token: ${{secrets.GITHUB_TOKEN }} + path-to-lcov: ${{steps.coverlet.outputs.coverageFile}} + - name: ReportGenerator + uses: danielpalme/ReportGenerator-GitHub-Action@4.8.12 + with: + reports: ${{steps.coverlet.outputs.coverageFile}} + targetdir: 'coveragereport' + reporttypes: 'HtmlInline;MarkdownSummary' + verbosity: 'Info' # The verbosity level of the log messages. Values: Verbose, Info, Warning, Error, Off + tag: '${{ github.run_number }}_${{ github.run_id }}' +# - name: Publish coverage summary +# uses: marocchino/sticky-pull-request-comment@v2 +# with: +# path: coveragereport/Summary.md + - name: publish nuget + if: matrix.os == 'windows-latest' && env.PUBLISH_NUGET + id: publish_nuget + uses: brandedoutcast/publish-nuget@v2.5.2 + with: + VERSION_REGEX: (.*)<\/version> + PROJECT_FILE_PATH: ${{env.MAIN_CSPROJ}} + NUGET_KEY: ${{secrets.NUGET_KEY}} + VERSION_FILE_PATH: ${{env.MAIN_CSPROJ}} + - name: Create Release + if: ${{ success() && matrix.os == 'windows-latest' && steps.publish_nuget.outputs.VERSION != '' && steps.publish_nuget.outputs.VERSION != null }} + id: create_release + uses: actions/create-release@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ steps.publish_nuget.outputs.VERSION }} + release_name: Release ${{ steps.publish_nuget.outputs.VERSION }} + draft: false + prerelease: false + - name: Upload Release Asset + if: ${{ success() && matrix.os == 'windows-latest' && steps.create_release.outputs.upload_url != '' && steps.create_release.outputs.upload_url != null }} + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ${{ steps.publish_nuget.outputs.PACKAGE_PATH }} + asset_name: ${{ steps.publish_nuget.outputs.PACKAGE_NAME }} + asset_content_type: application/zip + - name: refresh coverage badge + uses: fjogeleit/http-request-action@master + with: + url: https://camo.githubusercontent.com/12c4fcb3b21fbb2a725fc61449fb1b91e972c4c8a2baaf5904936d8e334bdbe8/68747470733a2f2f636f766572616c6c732e696f2f7265706f732f6769746875622f62336230302f63736c792f62616467652e7376673f6272616e63683d64657626736572766963653d676974687562 + method: PURGE + - name: refresh nuget badge + uses: fjogeleit/http-request-action@master + with: + url: https://camo.githubusercontent.com/b0c8ccfcb3256380ae37c312e789e6282143295fa495361ea5c2dbe2169bff8c/68747470733a2f2f696d672e736869656c64732e696f2f6e756765742f762f736c792e737667 + method: PURGE diff --git a/ParserTests/ImplicitTokensTests.cs b/ParserTests/ImplicitTokensTests.cs index 96851d42..45daf5b1 100644 --- a/ParserTests/ImplicitTokensTests.cs +++ b/ParserTests/ImplicitTokensTests.cs @@ -119,9 +119,11 @@ public void BuildParserTest() var parser = BuildParser(); Assert.True(parser.IsOk); Assert.NotNull(parser.Result); - var r = parser.Result.Parse("1.0 + 2.0 + bozzo"); + var r = parser.Result.Parse("2.0 - 2.0 + bozzo + Test"); Assert.True(r.IsOk); - Assert.Equal(45.0,r.Result); + // grammar is left associative so expression really is + // (2.0 - (2.0 + (bozzo + Test))) = 2 - ( 2 + (42 + 0)) = 2 - (2 + 42) = 2 - 44 = -42 + Assert.Equal(-42.0,r.Result); } } From 2c5fe6da9ac4a4fb7031184170f4d17ea87e22cc Mon Sep 17 00:00:00 2001 From: b3b00 Date: Mon, 6 Jun 2022 18:49:52 +0200 Subject: [PATCH 07/22] implicit token , expression parser : initial --- ParserTests/ImplicitTokensExpressionParser.cs | 101 +++++++++++++++++ ParserTests/ImplicitTokensParser.cs | 63 +++++++++++ ParserTests/ImplicitTokensTests.cs | 103 ++++-------------- ParserTests/ImplicitTokensTokens.cs | 28 +++++ sly/EnumConverter.cs | 25 +++++ .../generator/ExpressionRulesGenerator.cs | 35 +++++- 6 files changed, 271 insertions(+), 84 deletions(-) create mode 100644 ParserTests/ImplicitTokensExpressionParser.cs create mode 100644 ParserTests/ImplicitTokensParser.cs create mode 100644 ParserTests/ImplicitTokensTokens.cs diff --git a/ParserTests/ImplicitTokensExpressionParser.cs b/ParserTests/ImplicitTokensExpressionParser.cs new file mode 100644 index 00000000..f3f39ee2 --- /dev/null +++ b/ParserTests/ImplicitTokensExpressionParser.cs @@ -0,0 +1,101 @@ +using expressionparser; +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests +{ + public class ImplicitTokensExpressionParser + { + [Production("primary: DOUBLE")] + [Operand] + public double Primary(Token doubleToken) + { + return doubleToken.DoubleValue; + } + + [Operand] + [Production("primary : 'bozzo'[d]")] + public double Bozzo() + { + return 42.0; + } + + [Operand] + [Production("primary : TEST[d]")] + public double Test() + { + return 0.0; + } + + + [Operation("+", Affix.InFix, Associativity.Right, 10)] + [Operation("-", Affix.InFix, Associativity.Left, 10)] + public double BinaryTermExpression(double left, Token operation, double right) + { + return 0; + } + + [Operation((int) ImplicitTokensTokens.TIMES, Affix.InFix, Associativity.Right, 50)] + [Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)] + public double BinaryFactorExpression(double left, Token operation, double right) + { + double result = 0; + switch (operation.TokenID) + { + case ExpressionToken.TIMES: + { + result = left * right; + break; + } + case ExpressionToken.DIVIDE: + { + result = left / right; + break; + } + } + + return result; + } + + + [Operation("-", Affix.PreFix, Associativity.Right, 100)] + public double PreFixExpression(Token operation, double value) + { + return -value; + } + + + public double Expression(double left, Token operatorToken, double right) + { + double result = 0.0; + + + switch (operatorToken.StringWithoutQuotes) + { + case "+": + { + result = left + right; + break; + } + case "-": + { + result = left - right; + break; + } + } + + return result; + } + + + [Production("expression : primary ")] + public double Simple(double value) + { + return value; + } + + + + + } +} \ No newline at end of file diff --git a/ParserTests/ImplicitTokensParser.cs b/ParserTests/ImplicitTokensParser.cs new file mode 100644 index 00000000..5c5660e7 --- /dev/null +++ b/ParserTests/ImplicitTokensParser.cs @@ -0,0 +1,63 @@ +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests +{ + public class ImplicitTokensParser + { + [Production("primary: DOUBLE")] + public double Primary(Token doubleToken) + { + return doubleToken.DoubleValue; + } + + [Production("primary : 'bozzo'[d]")] + public double Bozzo() + { + return 42.0; + } + + [Production("primary : TEST[d]")] + public double Test() + { + return 0.0; + } + + + [Production("expression : primary ['+' | '-'] expression")] + + + public double Expression(double left, Token operatorToken, double right) + { + double result = 0.0; + + + switch (operatorToken.StringWithoutQuotes) + { + case "+": + { + result = left + right; + break; + } + case "-": + { + result = left - right; + break; + } + } + + return result; + } + + + [Production("expression : primary ")] + public double Simple(double value) + { + return value; + } + + + + + } +} \ No newline at end of file diff --git a/ParserTests/ImplicitTokensTests.cs b/ParserTests/ImplicitTokensTests.cs index 45daf5b1..bdee08b2 100644 --- a/ParserTests/ImplicitTokensTests.cs +++ b/ParserTests/ImplicitTokensTests.cs @@ -3,13 +3,11 @@ using System.IO; using System.Linq; using System.Text; -using expressionparser; using indented; using jsonparser; using jsonparser.JsonModel; using simpleExpressionParser; using sly.buildresult; -using sly.lexer; using sly.parser; using sly.parser.generator; using sly.parser.generator.visitor; @@ -21,86 +19,6 @@ namespace ParserTests { - - [Lexer(IgnoreWS = true, IgnoreEOL = true)] - public enum ImplicitTokensTokens - { - [MultiLineComment("/*","*/")] - MULTILINECOMMENT = 1, - - [SingleLineComment("//")] - SINGLELINECOMMENT = 2, - - [Lexeme(GenericToken.Identifier, IdentifierType.AlphaNumeric)] - ID = 3, - - [Lexeme(GenericToken.Double, channel:101)] - DOUBLE = 4, - - [Keyword("Test")] - TEST = 5 - - } - - public class ImplicitTokensParser - { - [Production("primary: DOUBLE")] - public double Primary(Token doubleToken) - { - return doubleToken.DoubleValue; - } - - [Production("primary : 'bozzo'[d]")] - public double Bozzo() - { - return 42.0; - } - - [Production("primary : TEST[d]")] - public double Test() - { - return 0.0; - } - - - [Production("expression : primary ['+' | '-'] expression")] - - - public double Expression(double left, Token operatorToken, double right) - { - double result = 0.0; - - - switch (operatorToken.StringWithoutQuotes) - { - case "+": - { - result = left + right; - break; - } - case "-": - { - result = left - right; - break; - } - } - - return result; - } - - - [Production("expression : primary ")] - public double Simple(double value) - { - return value; - } - - - - - } - - public class ImplicitTokensTests { private Parser Parser; @@ -112,6 +30,14 @@ public class ImplicitTokensTests var result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "expression"); return result; } + + private BuildResult> BuildExpressionParser() + { + var parserInstance = new ImplicitTokensExpressionParser(); + var builder = new ParserBuilder(); + var result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, nameof(ImplicitTokensExpressionParser)+"_expression"); + return result; + } [Fact] public void BuildParserTest() @@ -125,6 +51,19 @@ public void BuildParserTest() // (2.0 - (2.0 + (bozzo + Test))) = 2 - ( 2 + (42 + 0)) = 2 - (2 + 42) = 2 - 44 = -42 Assert.Equal(-42.0,r.Result); } + + [Fact] + public void BuildExpressionParserTest() + { + var parser = BuildExpressionParser(); + Assert.True(parser.IsOk); + Assert.NotNull(parser.Result); + var r = parser.Result.Parse("2.0 - 2.0 + bozzo + Test"); + Assert.True(r.IsOk); + + + Assert.Equal(2 - 2 + 42 + 0,r.Result); + } } } diff --git a/ParserTests/ImplicitTokensTokens.cs b/ParserTests/ImplicitTokensTokens.cs new file mode 100644 index 00000000..13f5c8ad --- /dev/null +++ b/ParserTests/ImplicitTokensTokens.cs @@ -0,0 +1,28 @@ +using sly.lexer; + +namespace ParserTests +{ + [Lexer(IgnoreWS = true, IgnoreEOL = true)] + public enum ImplicitTokensTokens + { + [MultiLineComment("/*","*/")] + MULTILINECOMMENT = 1, + + [SingleLineComment("//")] + SINGLELINECOMMENT = 2, + + [Lexeme(GenericToken.Identifier, IdentifierType.AlphaNumeric)] + ID = 3, + + [Lexeme(GenericToken.Double, channel:101)] + DOUBLE = 4, + + [Keyword("Test")] + TEST = 5, + [Sugar("*")] + TIMES = 6, + + [Sugar("/")] + DIVIDE = 7 + } +} \ No newline at end of file diff --git a/sly/EnumConverter.cs b/sly/EnumConverter.cs index cfdd7fe4..a0a1e0b3 100644 --- a/sly/EnumConverter.cs +++ b/sly/EnumConverter.cs @@ -21,6 +21,25 @@ public static IN ConvertIntToEnum(int intValue) return default(IN); } + + public static bool IsEnumValue(int intValue) + { + var genericType = typeof(IN); + if (genericType.IsEnum) + foreach (IN value in Enum.GetValues(genericType)) + { + var test = Enum.Parse(typeof(IN), value.ToString()) as Enum; + var val = Convert.ToInt32(test); + if (val == intValue) + { + return true; + } + } + + return false; + } + + public static IN ConvertStringToEnum(string name) where IN : struct { @@ -31,5 +50,11 @@ public static IN ConvertIntToEnum(int intValue) } return token; } + + public static bool IsEnumValue(string name) where IN : struct + { + + return Enum.TryParse(name, out IN token); + } } } \ No newline at end of file diff --git a/sly/parser/generator/ExpressionRulesGenerator.cs b/sly/parser/generator/ExpressionRulesGenerator.cs index bc72848b..dacdce5e 100644 --- a/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/sly/parser/generator/ExpressionRulesGenerator.cs @@ -20,6 +20,15 @@ public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, OperatorToken = oper; Affix = affix; } + + public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, Affix affix, string oper) + { + Precedence = precedence; + Associativity = assoc; + VisitorMethod = method; + ImplicitOperatorToken = oper; + Affix = affix; + } public int Precedence { get; set; } @@ -35,6 +44,10 @@ public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, public bool IsUnary => Affix != Affix.InFix; + public bool IsImplicitOperatorToken => !string.IsNullOrEmpty(ImplicitOperatorToken); + + public string ImplicitOperatorToken { get; set; } + [ExcludeFromCodeCoverage] public override string ToString() { @@ -77,15 +90,33 @@ public ExpressionRulesGenerator(string i18n = null) foreach (var attr in attributes) { IN oper = default; + string implicitToken = null; if (attr.IsIntToken) { oper = EnumConverter.ConvertIntToEnum(attr.IntToken); } else if (attr.IsStringToken) { - oper = EnumConverter.ConvertStringToEnum(attr.StringToken); + if (EnumConverter.IsEnumValue(attr.StringToken)) + { + oper = EnumConverter.ConvertStringToEnum(attr.StringToken); + } + else + { + implicitToken = attr.StringToken; + } + } + + OperationMetaData operation = null; + if (string.IsNullOrEmpty(implicitToken)) + { + operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, oper); + } + else + { + operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, implicitToken); } - var operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, oper); + var operations = new List>(); if (operationsByPrecedence.ContainsKey(operation.Precedence)) operations = operationsByPrecedence[operation.Precedence]; From ce19c86ed5ee307d0ca967145a1cec53798257a3 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Mon, 6 Jun 2022 20:16:08 +0200 Subject: [PATCH 08/22] wip : implicit tokens and expressions --- ParserTests/ImplicitTokensExpressionParser.cs | 19 ++++++++++----- ParserTests/ImplicitTokensTests.cs | 2 +- .../generator/ExpressionRulesGenerator.cs | 12 +++++++++- sly/parser/generator/ParserBuilder.cs | 24 ++++++++++++++++++- sly/parser/generator/ParserConfiguration.cs | 8 ++++++- sly/parser/syntax/grammar/TerminalClause.cs | 4 ++++ 6 files changed, 59 insertions(+), 10 deletions(-) diff --git a/ParserTests/ImplicitTokensExpressionParser.cs b/ParserTests/ImplicitTokensExpressionParser.cs index f3f39ee2..34bce728 100644 --- a/ParserTests/ImplicitTokensExpressionParser.cs +++ b/ParserTests/ImplicitTokensExpressionParser.cs @@ -1,3 +1,4 @@ +using System; using expressionparser; using sly.lexer; using sly.parser.generator; @@ -28,26 +29,32 @@ public double Test() } - [Operation("+", Affix.InFix, Associativity.Right, 10)] + [Operation("+", Affix.InFix, Associativity.Left, 10)] [Operation("-", Affix.InFix, Associativity.Left, 10)] public double BinaryTermExpression(double left, Token operation, double right) { + switch (operation.Value) + { + case "+" : return left + right; + case "-" : return left - right; + default : throw new InvalidOperationException($"that is not possible ! {operation.Value} is not a valid operation"); + } return 0; } - [Operation((int) ImplicitTokensTokens.TIMES, Affix.InFix, Associativity.Right, 50)] + [Operation((int) ImplicitTokensTokens.TIMES, Affix.InFix, Associativity.Left, 50)] [Operation("DIVIDE", Affix.InFix, Associativity.Left, 50)] - public double BinaryFactorExpression(double left, Token operation, double right) + public double BinaryFactorExpression(double left, Token operation, double right) { double result = 0; switch (operation.TokenID) { - case ExpressionToken.TIMES: + case ImplicitTokensTokens.TIMES: { result = left * right; break; } - case ExpressionToken.DIVIDE: + case ImplicitTokensTokens.DIVIDE: { result = left / right; break; @@ -59,7 +66,7 @@ public double BinaryFactorExpression(double left, Token operati [Operation("-", Affix.PreFix, Associativity.Right, 100)] - public double PreFixExpression(Token operation, double value) + public double PreFixExpression(Token operation, double value) { return -value; } diff --git a/ParserTests/ImplicitTokensTests.cs b/ParserTests/ImplicitTokensTests.cs index bdee08b2..54c8211d 100644 --- a/ParserTests/ImplicitTokensTests.cs +++ b/ParserTests/ImplicitTokensTests.cs @@ -35,7 +35,7 @@ public class ImplicitTokensTests { var parserInstance = new ImplicitTokensExpressionParser(); var builder = new ParserBuilder(); - var result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, nameof(ImplicitTokensExpressionParser)+"_expression"); + var result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, nameof(ImplicitTokensExpressionParser)+"_expressions"); return result; } diff --git a/sly/parser/generator/ExpressionRulesGenerator.cs b/sly/parser/generator/ExpressionRulesGenerator.cs index dacdce5e..e6c11155 100644 --- a/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/sly/parser/generator/ExpressionRulesGenerator.cs @@ -249,7 +249,17 @@ private NonTerminal BuildPrecedenceNonTerminal(string name, string nextName, var InFixOps = operations.Where>(x => x.Affix == Affix.InFix).ToList>(); if (InFixOps.Count > 0) { - var InFixClauses = InFixOps.Select, TerminalClause>(x => new TerminalClause(x.OperatorToken)).ToList>(); + var InFixClauses = InFixOps.Select, TerminalClause>(x => + { + if (x.IsImplicitOperatorToken) + { + return new TerminalClause(x.ImplicitOperatorToken); + } + else + { + return new TerminalClause(x.OperatorToken); + } + }).ToList>(); var rule = new Rule { diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index acf628ae..cd7f75d7 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -102,8 +102,30 @@ public ParserBuilder() : this(null) { var expressionResult = parser.BuildExpressionParser(result, rootRule); if (expressionResult.IsError) result.AddErrors(expressionResult.Errors); + + + result.Result.Configuration = expressionResult.Result; - + + // TODO : how to avoid this double lexer initialization ? + var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess, result.Result.Configuration); + if (lexerResult.IsError) + { + foreach (var lexerResultError in lexerResult.Errors) + { + result.AddError(lexerResultError); + } + return result; + } + else + { + parser.Lexer = lexerResult.Result; + parser.Instance = parserInstance; + result.Result = parser; + return result; + } + +// todo : ??? result = CheckParser(result); if (result.IsError) { diff --git a/sly/parser/generator/ParserConfiguration.cs b/sly/parser/generator/ParserConfiguration.cs index 45179881..1503bc5d 100644 --- a/sly/parser/generator/ParserConfiguration.cs +++ b/sly/parser/generator/ParserConfiguration.cs @@ -43,7 +43,13 @@ public List> GetAllImplicitTokenClauses() clauses.Add(terminal); } } - } + } + + if (clause is OptionClause option) + { + if (option.Clause is TerminalClause terminal && terminal.IsImplicitToken) + clauses.Add(terminal); + } } } } diff --git a/sly/parser/syntax/grammar/TerminalClause.cs b/sly/parser/syntax/grammar/TerminalClause.cs index a4fc7886..69508ee4 100644 --- a/sly/parser/syntax/grammar/TerminalClause.cs +++ b/sly/parser/syntax/grammar/TerminalClause.cs @@ -21,6 +21,10 @@ public TerminalClause(string implicitToken, bool discard) : this(default(T)) ImplicitToken = implicitToken; Discarded = discard; } + + public TerminalClause(string implicitToken) : this(implicitToken,false) + { + } public T ExpectedToken { get; set; } From 1e7c9bcd72b62ac651752553a48d0f4bce36e067 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 7 Jun 2022 09:09:29 +0200 Subject: [PATCH 09/22] implicit token in operations : working but making fail other unit tests --- sly/lexer/Token.cs | 5 ++ .../generator/ExpressionRulesGenerator.cs | 48 +----------------- sly/parser/generator/OperationMetaData.cs | 50 +++++++++++++++++++ sly/parser/syntax/grammar/TerminalClause.cs | 4 ++ 4 files changed, 60 insertions(+), 47 deletions(-) create mode 100644 sly/parser/generator/OperationMetaData.cs diff --git a/sly/lexer/Token.cs b/sly/lexer/Token.cs index 7372c694..912eb302 100644 --- a/sly/lexer/Token.cs +++ b/sly/lexer/Token.cs @@ -224,6 +224,11 @@ public override string ToString() { value = $"<>"; } + + if (IsImplicit) + { + value = $"[{Value.Replace("\r", "").Replace("\n", "")}]"; + } return $"{value} @{Position} on channel {Channel}"; diff --git a/sly/parser/generator/ExpressionRulesGenerator.cs b/sly/parser/generator/ExpressionRulesGenerator.cs index e6c11155..9bb0e414 100644 --- a/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/sly/parser/generator/ExpressionRulesGenerator.cs @@ -1,5 +1,4 @@ -using System.Diagnostics.CodeAnalysis; -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -10,51 +9,6 @@ namespace sly.parser.generator { - public class OperationMetaData where T : struct - { - public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, Affix affix, T oper) - { - Precedence = precedence; - Associativity = assoc; - VisitorMethod = method; - OperatorToken = oper; - Affix = affix; - } - - public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, Affix affix, string oper) - { - Precedence = precedence; - Associativity = assoc; - VisitorMethod = method; - ImplicitOperatorToken = oper; - Affix = affix; - } - - public int Precedence { get; set; } - - public Associativity Associativity { get; set; } - - public MethodInfo VisitorMethod { get; set; } - - public T OperatorToken { get; set; } - - public Affix Affix { get; set; } - - public bool IsBinary => Affix == Affix.InFix; - - public bool IsUnary => Affix != Affix.InFix; - - public bool IsImplicitOperatorToken => !string.IsNullOrEmpty(ImplicitOperatorToken); - - public string ImplicitOperatorToken { get; set; } - - [ExcludeFromCodeCoverage] - public override string ToString() - { - return $"{OperatorToken} / {Affix} : {Precedence} / {Associativity}"; - } - } - public class ExpressionRulesGenerator where IN : struct { public string I18n { get; set; } diff --git a/sly/parser/generator/OperationMetaData.cs b/sly/parser/generator/OperationMetaData.cs new file mode 100644 index 00000000..13e4fcf0 --- /dev/null +++ b/sly/parser/generator/OperationMetaData.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.CodeAnalysis; +using System.Reflection; + +namespace sly.parser.generator +{ + public class OperationMetaData where T : struct + { + public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, Affix affix, T oper) + { + Precedence = precedence; + Associativity = assoc; + VisitorMethod = method; + OperatorToken = oper; + Affix = affix; + } + + public OperationMetaData(int precedence, Associativity assoc, MethodInfo method, Affix affix, string oper) + { + Precedence = precedence; + Associativity = assoc; + VisitorMethod = method; + ImplicitOperatorToken = oper; + Affix = affix; + } + + public int Precedence { get; set; } + + public Associativity Associativity { get; set; } + + public MethodInfo VisitorMethod { get; set; } + + public T OperatorToken { get; set; } + + public Affix Affix { get; set; } + + public bool IsBinary => Affix == Affix.InFix; + + public bool IsUnary => Affix != Affix.InFix; + + public bool IsImplicitOperatorToken => !string.IsNullOrEmpty(ImplicitOperatorToken); + + public string ImplicitOperatorToken { get; set; } + + [ExcludeFromCodeCoverage] + public override string ToString() + { + return $"{OperatorToken} / {Affix} : {Precedence} / {Associativity}"; + } + } +} \ No newline at end of file diff --git a/sly/parser/syntax/grammar/TerminalClause.cs b/sly/parser/syntax/grammar/TerminalClause.cs index 69508ee4..28c8ba01 100644 --- a/sly/parser/syntax/grammar/TerminalClause.cs +++ b/sly/parser/syntax/grammar/TerminalClause.cs @@ -41,6 +41,10 @@ public virtual bool MayBeEmpty() public virtual bool Check(Token nextToken) { + if (IsImplicitToken) + { + return nextToken.Value.Equals(ImplicitToken); + } return nextToken.TokenID.Equals(ExpectedToken); } From 11aaecea9df07dbc895eb43af2a5651c6f72709d Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 7 Jun 2022 09:53:36 +0200 Subject: [PATCH 10/22] implicit token in operations : debugging --- ParserTests/ParserConfigurationTests.cs | 12 ++++++++---- sly/parser/parser/ISyntaxParser.cs | 2 ++ .../llparser/RecursiveDescentSyntaxParser.cs | 9 +++++++-- .../RecursiveDescentSyntaxParserStarter.cs | 5 +++++ sly/parser/syntax/grammar/ChoiceClause.cs | 3 ++- sly/parser/syntax/grammar/NonTerminalClause.cs | 2 +- sly/parser/syntax/grammar/TerminalClause.cs | 18 ++++++++++++++++-- sly/parser/syntax/grammar/ZeroOrMoreClause.cs | 3 ++- 8 files changed, 43 insertions(+), 11 deletions(-) diff --git a/ParserTests/ParserConfigurationTests.cs b/ParserTests/ParserConfigurationTests.cs index c592ba56..f554a9f9 100644 --- a/ParserTests/ParserConfigurationTests.cs +++ b/ParserTests/ParserConfigurationTests.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using expressionparser; using sly.buildresult; @@ -391,14 +392,17 @@ public void TestBadManyArgument() var instance = new BadManyArgParser(); ParserBuilder builder = new ParserBuilder("en"); var result = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "badmanyarg"); + var d = result.Result.SyntaxParser.Dump(); + Debug.WriteLine(d); + Assert.True(result.IsError); Assert.Equal(4,result.Errors.Count); //"visitor BadReturn for rule badtermarg : A B ; parameter a has incorrect type : expected sly.lexer.Token`1[ParserTests.BadVisitorTokens], found SubBadVisitor" - Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter aArg has incorrect type"))); - Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter bArg has incorrect type"))); - Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter cArg has incorrect type"))); - Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter dArg has incorrect type"))); + Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter aArg has incorrect type") && x.Code == ErrorCodes.PARSER_INCORRECT_VISITOR_PARAMETER_TYPE)); + Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter bArg has incorrect type") && x.Code == ErrorCodes.PARSER_INCORRECT_VISITOR_PARAMETER_TYPE)); + Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter cArg has incorrect type") && x.Code == ErrorCodes.PARSER_INCORRECT_VISITOR_PARAMETER_TYPE)); + Assert.True(result.Errors.Exists(x => x.Message.Contains("parameter dArg has incorrect type") && x.Code == ErrorCodes.PARSER_INCORRECT_VISITOR_PARAMETER_TYPE)); } diff --git a/sly/parser/parser/ISyntaxParser.cs b/sly/parser/parser/ISyntaxParser.cs index f13bfc37..7aa6f088 100644 --- a/sly/parser/parser/ISyntaxParser.cs +++ b/sly/parser/parser/ISyntaxParser.cs @@ -11,5 +11,7 @@ public interface ISyntaxParser where IN : struct SyntaxParseResult Parse(IList> tokens, string startingNonTerminal = null); void Init(ParserConfiguration configuration, string root); + + string Dump(); } } \ No newline at end of file diff --git a/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs b/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs index d2253bf4..f96b625d 100644 --- a/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs +++ b/sly/parser/parser/llparser/RecursiveDescentSyntaxParser.cs @@ -249,8 +249,11 @@ public SyntaxParseResult ParseTerminal(IList> tokens, TerminalClau token.Discarded = terminal.Discarded; result.Root = new SyntaxLeaf(token, terminal.Discarded); result.HasByPassNodes = false; - result.Errors.Add(new UnexpectedTokenSyntaxError(token,I18n,terminal.ExpectedToken)); - result.AddExpecting(terminal.ExpectedToken); + if (result.IsError) + { + result.Errors.Add(new UnexpectedTokenSyntaxError(token, I18n, terminal.ExpectedToken)); + result.AddExpecting(terminal.ExpectedToken); + } return result; } @@ -415,6 +418,8 @@ public virtual void Init(ParserConfiguration configuration, string root InitializeStartingTokens(configuration, StartingNonTerminal); } + + #endregion } } \ No newline at end of file diff --git a/sly/parser/parser/llparser/RecursiveDescentSyntaxParserStarter.cs b/sly/parser/parser/llparser/RecursiveDescentSyntaxParserStarter.cs index f4286000..a06e48b9 100644 --- a/sly/parser/parser/llparser/RecursiveDescentSyntaxParserStarter.cs +++ b/sly/parser/parser/llparser/RecursiveDescentSyntaxParserStarter.cs @@ -165,5 +165,10 @@ protected virtual void InitializeStartingTokens(ParserConfiguration con #endregion + public string Dump() + { + return this.Configuration.Dump(); + } + } } \ No newline at end of file diff --git a/sly/parser/syntax/grammar/ChoiceClause.cs b/sly/parser/syntax/grammar/ChoiceClause.cs index 1431ddeb..7d606e44 100644 --- a/sly/parser/syntax/grammar/ChoiceClause.cs +++ b/sly/parser/syntax/grammar/ChoiceClause.cs @@ -31,7 +31,8 @@ public ChoiceClause(IClause choice, List> choices) : this(choice) [ExcludeFromCodeCoverage] public override string ToString() { - return string.Join(" | ", Choices.Select, string>(c => c.ToString())); + var choices = string.Join(" | ", Choices.Select, string>(c => c.Dump())); + return $"[ {choices} ]"; } public bool MayBeEmpty() diff --git a/sly/parser/syntax/grammar/NonTerminalClause.cs b/sly/parser/syntax/grammar/NonTerminalClause.cs index 92eb5136..85fb9ac5 100644 --- a/sly/parser/syntax/grammar/NonTerminalClause.cs +++ b/sly/parser/syntax/grammar/NonTerminalClause.cs @@ -28,7 +28,7 @@ public override string ToString() [ExcludeFromCodeCoverage] public string Dump() { - return NonTerminalName; + return $"{NonTerminalName}(NT)"; } } } \ No newline at end of file diff --git a/sly/parser/syntax/grammar/TerminalClause.cs b/sly/parser/syntax/grammar/TerminalClause.cs index 28c8ba01..b74b2148 100644 --- a/sly/parser/syntax/grammar/TerminalClause.cs +++ b/sly/parser/syntax/grammar/TerminalClause.cs @@ -52,14 +52,28 @@ public virtual bool Check(Token nextToken) public override string ToString() { var b = new StringBuilder(); - b.Append(ExpectedToken); + if (IsImplicitToken) + { + b.Append($"'{ImplicitToken}'"); + } + else + { + b.Append(ExpectedToken); + } + if (Discarded) b.Append("[d]"); + b.Append("(T)"); return b.ToString(); } public virtual string Dump() { - return ExpectedToken.ToString(); + + if (IsImplicitToken) + { + return $"'{ImplicitToken}'(T)"; + } + return $"{ExpectedToken}(T)"; } } diff --git a/sly/parser/syntax/grammar/ZeroOrMoreClause.cs b/sly/parser/syntax/grammar/ZeroOrMoreClause.cs index f6829819..c97d2818 100644 --- a/sly/parser/syntax/grammar/ZeroOrMoreClause.cs +++ b/sly/parser/syntax/grammar/ZeroOrMoreClause.cs @@ -24,7 +24,8 @@ public override bool MayBeEmpty() [ExcludeFromCodeCoverage] public override string Dump() { - return Clause.Dump()+"*"; + var t = Clause.Dump() + "*"; + return t; } } } \ No newline at end of file From bdc7b1596c8ddf28313ff628bb4b99335e52a969 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 7 Jun 2022 17:57:54 +0200 Subject: [PATCH 11/22] fix --- ParserTests/ParserConfigurationTests.cs | 2 -- sly/parser/generator/ParserBuilder.cs | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ParserTests/ParserConfigurationTests.cs b/ParserTests/ParserConfigurationTests.cs index f554a9f9..56596ee2 100644 --- a/ParserTests/ParserConfigurationTests.cs +++ b/ParserTests/ParserConfigurationTests.cs @@ -392,8 +392,6 @@ public void TestBadManyArgument() var instance = new BadManyArgParser(); ParserBuilder builder = new ParserBuilder("en"); var result = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "badmanyarg"); - var d = result.Result.SyntaxParser.Dump(); - Debug.WriteLine(d); Assert.True(result.IsError); Assert.Equal(4,result.Errors.Count); diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index cd7f75d7..0de0431e 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -115,22 +115,21 @@ public ParserBuilder() : this(null) { result.AddError(lexerResultError); } - return result; } else { parser.Lexer = lexerResult.Result; parser.Instance = parserInstance; result.Result = parser; - return result; + } -// todo : ??? result = CheckParser(result); if (result.IsError) { result.Result = null; } + return result; } else { From 6f873f4ec433769c2e00d116ebeb61f959540c4f Mon Sep 17 00:00:00 2001 From: b3b00 Date: Tue, 7 Jun 2022 20:57:17 +0200 Subject: [PATCH 12/22] cleaning --- sly/lexer/LexerBuilder.cs | 1 - sly/parser/generator/ExpressionRulesGenerator.cs | 2 +- sly/parser/generator/ParserBuilder.cs | 3 --- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/sly/lexer/LexerBuilder.cs b/sly/lexer/LexerBuilder.cs index 221aa23a..dddc929d 100644 --- a/sly/lexer/LexerBuilder.cs +++ b/sly/lexer/LexerBuilder.cs @@ -149,7 +149,6 @@ private static bool IsGenericLexer(Dictionary> att { foreach (var lexeme in lexemes) { - // TODO ??? var channel = lexeme.Channel.HasValue ? lexeme.Channel.Value : 0; lexer.AddDefinition(new TokenDefinition(tokenID, lexeme.Pattern, channel,lexeme.IsSkippable, lexeme.IsLineEnding)); diff --git a/sly/parser/generator/ExpressionRulesGenerator.cs b/sly/parser/generator/ExpressionRulesGenerator.cs index 9bb0e414..7e85d282 100644 --- a/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/sly/parser/generator/ExpressionRulesGenerator.cs @@ -62,7 +62,7 @@ public ExpressionRulesGenerator(string i18n = null) } OperationMetaData operation = null; - if (string.IsNullOrEmpty(implicitToken)) + if (string.IsNullOrEmpty(implicitToken)) // TODO ?? uniquement si contient '...' ?? { operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, oper); } diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index 0de0431e..a1d36235 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -207,7 +207,6 @@ public ParserBuilder() : this(null) var x = genericLexer.FSMBuilder.Fsm.Run(@implicit, new LexerPosition()); if (x.IsSuccess) { - // todo : if x.NodeId == "in_identifier" var t = genericLexer.FSMBuilder.Marks; var y = genericLexer.FSMBuilder.Marks.FirstOrDefault(k => k.Value == x.NodeId); if (y.Key == GenericLexer.in_identifier) // implicit keyword @@ -224,9 +223,7 @@ public ParserBuilder() : this(null) genericLexer.AddSugarLexem(default(IN),result,@implicit); } } - ; } - ; // TODO } } From e022f18c0941ed3e1e4f966c43383c51b51bce45 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 8 Jun 2022 09:14:01 +0200 Subject: [PATCH 13/22] fix unit tests implicit tokens in operations --- ParserTests/ImplicitTokensExpressionParser.cs | 6 +++--- sly/parser/generator/ExpressionRulesGenerator.cs | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ParserTests/ImplicitTokensExpressionParser.cs b/ParserTests/ImplicitTokensExpressionParser.cs index 34bce728..c566e779 100644 --- a/ParserTests/ImplicitTokensExpressionParser.cs +++ b/ParserTests/ImplicitTokensExpressionParser.cs @@ -29,8 +29,8 @@ public double Test() } - [Operation("+", Affix.InFix, Associativity.Left, 10)] - [Operation("-", Affix.InFix, Associativity.Left, 10)] + [Operation("'+'", Affix.InFix, Associativity.Left, 10)] + [Operation("'-'", Affix.InFix, Associativity.Left, 10)] public double BinaryTermExpression(double left, Token operation, double right) { switch (operation.Value) @@ -65,7 +65,7 @@ public double BinaryFactorExpression(double left, Token op } - [Operation("-", Affix.PreFix, Associativity.Right, 100)] + [Operation("'-'", Affix.PreFix, Associativity.Right, 100)] public double PreFixExpression(Token operation, double value) { return -value; diff --git a/sly/parser/generator/ExpressionRulesGenerator.cs b/sly/parser/generator/ExpressionRulesGenerator.cs index 7e85d282..c6f81411 100644 --- a/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/sly/parser/generator/ExpressionRulesGenerator.cs @@ -61,14 +61,21 @@ public ExpressionRulesGenerator(string i18n = null) } } + + bool isEnumValue = EnumConverter.IsEnumValue(attr.StringToken) || + EnumConverter.IsEnumValue(attr.IntToken); OperationMetaData operation = null; - if (string.IsNullOrEmpty(implicitToken)) // TODO ?? uniquement si contient '...' ?? + if (!isEnumValue && !string.IsNullOrEmpty(implicitToken) && implicitToken.StartsWith("'") && implicitToken.EndsWith("'")) // TODO ?? uniquement si contient '...' ?? + { + operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, implicitToken); + } + else if (isEnumValue) { operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, oper); } else { - operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, implicitToken); + throw new ParserConfigurationException($"bad enum name {attr.StringToken} on Operation definition."); } var operations = new List>(); @@ -207,7 +214,7 @@ private NonTerminal BuildPrecedenceNonTerminal(string name, string nextName, { if (x.IsImplicitOperatorToken) { - return new TerminalClause(x.ImplicitOperatorToken); + return new TerminalClause(x.ImplicitOperatorToken.Substring(1,x.ImplicitOperatorToken.Length-2)); } else { From 2c3374f7f021f9e82ef0cdf50af8565413c8ef2f Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 8 Jun 2022 11:03:04 +0200 Subject: [PATCH 14/22] fix graphviz generation for operation --- .../visitor/GraphVizEBNFSyntaxTreeVisitor.cs | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs b/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs index 9f9a0775..798d0ffc 100644 --- a/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs +++ b/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs @@ -116,11 +116,6 @@ private DotNode Visit(OptionSyntaxNode node) private string GetNodeLabel(SyntaxNode node) { string label = node.Name; - if (node.IsExpressionNode) - { - label = node.Operation.OperatorToken.ToString(); - } - return label; } @@ -138,12 +133,7 @@ private DotNode Visit(SyntaxNode node) children.Add(v); } - if (node.IsByPassNode) - { - //result = children[0]; - } - else - { + result = Node(GetNodeLabel(node)); Graph.Add(result); @@ -159,10 +149,6 @@ private DotNode Visit(SyntaxNode node) Graph.Add(edge); } }); - - } - - return result; } From 4f2142d2bbc86e8fe329851a663ffdcd859b3291 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 8 Jun 2022 11:04:31 +0200 Subject: [PATCH 15/22] dump graphviz --- .../visitor/GraphVizEBNFSyntaxTreeVisitor.cs | 9 +++- .../generator/visitor/dotgraph/DotArrow.cs | 15 ++++--- .../generator/visitor/dotgraph/DotGraph.cs | 43 +++++++++++++++++++ 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs b/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs index 798d0ffc..ef418961 100644 --- a/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs +++ b/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs @@ -35,7 +35,14 @@ private DotNode Leaf(SyntaxLeaf leaf) private DotNode Leaf(IN type, string value) { string label = type.ToString(); - label += "\n"; + if (label == "0") + { + label = ""; + } + else + { + label += "\n"; + } var esc = value.Replace("\"", "\\\""); label += "\\\"" + esc + "\\\""; var node = new DotNode(NodeCounter.ToString()) diff --git a/sly/parser/generator/visitor/dotgraph/DotArrow.cs b/sly/parser/generator/visitor/dotgraph/DotArrow.cs index 7ad9ee03..e1153373 100644 --- a/sly/parser/generator/visitor/dotgraph/DotArrow.cs +++ b/sly/parser/generator/visitor/dotgraph/DotArrow.cs @@ -4,13 +4,13 @@ namespace sly.parser.generator.visitor.dotgraph { public class DotArrow : IDot { - private DotNode source; - private DotNode destination; + public DotNode Source { get; private set; } + public DotNode Destination { get; private set; } public DotArrow(DotNode src, DotNode dest) { - source = src; - destination = dest; + Source = src; + Destination = dest; } public string Attribute(string name, string value) @@ -32,12 +32,17 @@ public string Attribute(string name, double value) public string ToGraph() { var builder = new StringBuilder(); - builder.Append($"{source.Name}->{destination?.Name} [ "); + builder.Append($"{Source.Name}->{Destination?.Name} [ "); builder.Append(Attribute("arrowshape", ArrowHeadShape)); builder.Append("];"); return builder.ToString(); } + + public string ToString() + { + return ToGraph(); + } } } \ No newline at end of file diff --git a/sly/parser/generator/visitor/dotgraph/DotGraph.cs b/sly/parser/generator/visitor/dotgraph/DotGraph.cs index 232aafda..80ea225d 100644 --- a/sly/parser/generator/visitor/dotgraph/DotGraph.cs +++ b/sly/parser/generator/visitor/dotgraph/DotGraph.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Text; namespace sly.parser.generator.visitor.dotgraph @@ -45,5 +46,47 @@ public string Compile() builder.AppendLine("}"); return builder.ToString(); } + + public List FindRoots() + { + var roots = edges.Where(x => !edges.Any(y => y.Destination?.Name == x.Source?.Name)); + return roots.Select(x => x.Source).ToList(); + } + + public IList FindEgdes(DotNode node) + { + var nodeEdges = edges.Where(x => x.Source?.Name == node.Name); + return nodeEdges.ToList(); + } + + public string Dump() + { + var roots = FindRoots(); + return string.Join("\n\n",roots.Select(x => Dump("",x))); + } + + private string Dump(string tab, DotNode node) + { + if (node == null) + { + return ""; + } + StringBuilder builder = new StringBuilder(); + builder.Append(tab) + .AppendLine(node.Label); + var edges = FindEgdes(node); + if (edges != null && edges.Any()) + { + foreach (var edge in edges) + { + builder.AppendLine(Dump(tab + "\t", edge.Destination)); + } + } + return builder.ToString(); + } + + + + } } \ No newline at end of file From 64522bf6ca5c7e7be67689c1a48b807ecbf30256 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 8 Jun 2022 11:05:18 +0200 Subject: [PATCH 16/22] dump syntax tree to json --- sly/parser/syntax/tree/ISyntaxNode.cs | 3 ++ sly/parser/syntax/tree/SyntaxEpsilon.cs | 5 +++ sly/parser/syntax/tree/SyntaxLeaf.cs | 9 ++++- sly/parser/syntax/tree/SyntaxNode.cs | 46 +++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/sly/parser/syntax/tree/ISyntaxNode.cs b/sly/parser/syntax/tree/ISyntaxNode.cs index cb21cc95..cf08d93c 100644 --- a/sly/parser/syntax/tree/ISyntaxNode.cs +++ b/sly/parser/syntax/tree/ISyntaxNode.cs @@ -9,5 +9,8 @@ public interface ISyntaxNode where IN : struct bool HasByPassNodes { get; set; } string Dump(string tab); + + string ToJson(int index = 0); + } } \ No newline at end of file diff --git a/sly/parser/syntax/tree/SyntaxEpsilon.cs b/sly/parser/syntax/tree/SyntaxEpsilon.cs index 875adb8c..2b453acc 100644 --- a/sly/parser/syntax/tree/SyntaxEpsilon.cs +++ b/sly/parser/syntax/tree/SyntaxEpsilon.cs @@ -19,5 +19,10 @@ public string Dump(string tab) { return $"Epsilon"; } + + public string ToJson(int index = 0) + { + return $@"""{index}.Epsilon"":""e"""; + } } } \ No newline at end of file diff --git a/sly/parser/syntax/tree/SyntaxLeaf.cs b/sly/parser/syntax/tree/SyntaxLeaf.cs index 7e31be8f..dabde88d 100644 --- a/sly/parser/syntax/tree/SyntaxLeaf.cs +++ b/sly/parser/syntax/tree/SyntaxLeaf.cs @@ -20,7 +20,14 @@ public SyntaxLeaf(Token token, bool discarded) public string Dump(string tab) { - return $"{tab}+ {Token.TokenID} : {Token.Value} @{Token.PositionInTokenFlow}"; + return $"{tab}+ {Token.TokenID.ToString()} : {Token.Value} @{Token.PositionInTokenFlow}"; } + + public string ToJson(int index = 0) + { + return $@"""{index}.{Token.TokenID.ToString()}"" : ""{Token.Value}"""; + } + + } } \ No newline at end of file diff --git a/sly/parser/syntax/tree/SyntaxNode.cs b/sly/parser/syntax/tree/SyntaxNode.cs index efc737b4..bfa2527e 100644 --- a/sly/parser/syntax/tree/SyntaxNode.cs +++ b/sly/parser/syntax/tree/SyntaxNode.cs @@ -83,7 +83,23 @@ public ISyntaxNode Right public string Dump(string tab) { StringBuilder builder = new StringBuilder(); + string expressionSuffix = ""; + if (Operation != null && (Operation.IsBinary || Operation.IsBinary)) + { + if (Operation.IsImplicitOperatorToken) + { + expressionSuffix = Operation.ImplicitOperatorToken; + } + else + { + expressionSuffix = Operation.OperatorToken.ToString(); + } + + expressionSuffix = $">{expressionSuffix}<"; + } + builder.AppendLine($"{tab}+ {Name} {(IsByPassNode ? "===":"")}"); + foreach (var child in Children) { builder.AppendLine($"{child.Dump(tab + "\t")}"); @@ -91,6 +107,36 @@ public string Dump(string tab) return builder.ToString(); } + + public string ToJson(int index = 0) + { + StringBuilder builder = new StringBuilder(); + + + builder.Append($@"""{index}.{Name}"); + if (IsByPassNode) + { + builder.Append("--"); + } + + builder.AppendLine(@""" : {"); + + for (int i = 0; i < Children.Count; i++) + { + var child = Children[i]; + builder.Append(child.ToJson(i)); + if (i < Children.Count - 1) + { + builder.Append(","); + } + + builder.AppendLine(); + } + + builder.Append("}"); + + return builder.ToString(); + } #endregion From 5a8afdfc7a3fd4d98257325482b48e9c50d76f79 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 8 Jun 2022 11:06:09 +0200 Subject: [PATCH 17/22] operation implicit token unit tests --- ParserTests/ImplicitTokensTests.cs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/ParserTests/ImplicitTokensTests.cs b/ParserTests/ImplicitTokensTests.cs index 54c8211d..a33b3ab8 100644 --- a/ParserTests/ImplicitTokensTests.cs +++ b/ParserTests/ImplicitTokensTests.cs @@ -1,21 +1,9 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using indented; -using jsonparser; -using jsonparser.JsonModel; -using simpleExpressionParser; +using System.IO; using sly.buildresult; using sly.parser; using sly.parser.generator; using sly.parser.generator.visitor; -using sly.parser.llparser; -using sly.parser.parser; -using sly.parser.syntax.grammar; using Xunit; -using String = System.String; namespace ParserTests { From 1f010efb5c8a583d93424b8f66f18df779ea4483 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 8 Jun 2022 16:36:00 +0200 Subject: [PATCH 18/22] some cleaning --- ParserTests/ImplicitTokensTests.cs | 16 ++++++ sly/lexer/LexerBuilder.cs | 41 +++++++++++++-- sly/parser/generator/EBNFParserBuilder.cs | 2 +- .../generator/ExpressionRulesGenerator.cs | 2 +- sly/parser/generator/ParserBuilder.cs | 52 +++---------------- 5 files changed, 60 insertions(+), 53 deletions(-) diff --git a/ParserTests/ImplicitTokensTests.cs b/ParserTests/ImplicitTokensTests.cs index a33b3ab8..54d1e6c4 100644 --- a/ParserTests/ImplicitTokensTests.cs +++ b/ParserTests/ImplicitTokensTests.cs @@ -47,6 +47,22 @@ public void BuildExpressionParserTest() Assert.True(parser.IsOk); Assert.NotNull(parser.Result); var r = parser.Result.Parse("2.0 - 2.0 + bozzo + Test"); + var tree = r.SyntaxTree; + var graphviz = new GraphVizEBNFSyntaxTreeVisitor(); + var dump = tree.Dump("\t"); + // File.Delete(@"c:\temp\tree.txt"); + // File.WriteAllText(@"c:\temp\tree.txt",dump); + // + var json = $@"{{ +{tree.ToJson()} +}}"; + // File.Delete(@"c:\temp\tree.json"); + // File.WriteAllText(@"c:\temp\tree.json",json); + // + var root = graphviz.VisitTree(tree); + string graph = graphviz.Graph.Compile(); + // File.Delete("c:\\temp\\tree.dot"); + // File.AppendAllText("c:\\temp\\tree.dot", graph); Assert.True(r.IsOk); diff --git a/sly/lexer/LexerBuilder.cs b/sly/lexer/LexerBuilder.cs index dddc929d..a7ece483 100644 --- a/sly/lexer/LexerBuilder.cs +++ b/sly/lexer/LexerBuilder.cs @@ -85,12 +85,12 @@ public static class LexerBuilder public static BuildResult> BuildLexer(BuildResult> result, BuildExtension extensionBuilder = null, - string lang = null, LexerPostProcess lexerPostProcess = null) where IN : struct + string lang = null, LexerPostProcess lexerPostProcess = null, IList implicitTokens = null) where IN : struct { var attributes = GetLexemes(result,lang); if (!result.IsError) { - result = Build(attributes, result, extensionBuilder,lang); + result = Build(attributes, result, extensionBuilder,lang, implicitTokens); result.Result.LexerPostProcess = lexerPostProcess; } @@ -99,7 +99,8 @@ public static class LexerBuilder private static BuildResult> Build(Dictionary> attributes, - BuildResult> result, BuildExtension extensionBuilder = null, string lang = null) where IN : struct + BuildResult> result, BuildExtension extensionBuilder = null, string lang = null, + IList implicitTokens = null) where IN : struct { var hasRegexLexemes = IsRegexLexer(attributes); var hasGenericLexemes = IsGenericLexer(attributes); @@ -114,7 +115,7 @@ public static class LexerBuilder { if (hasRegexLexemes) result = BuildRegexLexer(attributes, result,lang); - else if (hasGenericLexemes) result = BuildGenericLexer(attributes, extensionBuilder, result, lang); + else if (hasGenericLexemes) result = BuildGenericLexer(attributes, extensionBuilder, result, lang, implicitTokens); } return result; @@ -257,7 +258,8 @@ private static IEnumerable ParseIdentifierPattern(string pattern) } private static BuildResult> BuildGenericLexer(Dictionary> attributes, - BuildExtension extensionBuilder, BuildResult> result, string lang) where IN : struct + BuildExtension extensionBuilder, BuildResult> result, string lang, + IList implicitTokens = null) where IN : struct { result = CheckStringAndCharTokens(attributes, result, lang); var (config, tokens) = GetConfigAndGenericTokens(attributes); @@ -321,6 +323,7 @@ private static IEnumerable ParseIdentifierPattern(string pattern) AddExtensions(Extensions, extensionBuilder, lexer); var comments = GetCommentsAttribute(result,lang); + if (!result.IsError) { foreach (var comment in comments) @@ -360,6 +363,34 @@ private static IEnumerable ParseIdentifierPattern(string pattern) } } } + + if (implicitTokens != null) + { + foreach (var implicitToken in implicitTokens) + { + var fsmBuilder = lexer.FSMBuilder; + var x = fsmBuilder.Fsm.Run(implicitToken, new LexerPosition()); + if (x.IsSuccess) + { + var t = fsmBuilder.Marks; + var y = fsmBuilder.Marks.FirstOrDefault(k => k.Value == x.NodeId); + if (y.Key == GenericLexer.in_identifier) // implicit keyword + { + var resultx = new BuildResult>(); + result.Errors.AddRange(resultx.Errors); + lexer.AddKeyWord(default(IN), implicitToken, resultx); + ; + } + } + else + { + var resulty = new BuildResult>(); + result.Errors.AddRange(resulty.Errors); + lexer.AddSugarLexem(default(IN), resulty, implicitToken); + } + } + } + } result.Result = lexer; diff --git a/sly/parser/generator/EBNFParserBuilder.cs b/sly/parser/generator/EBNFParserBuilder.cs index 29b5719f..667d4e52 100644 --- a/sly/parser/generator/EBNFParserBuilder.cs +++ b/sly/parser/generator/EBNFParserBuilder.cs @@ -72,7 +72,7 @@ public EBNFParserBuilder(string i18n = null) : base(i18n) } var parser = new Parser(I18n,syntaxParser, visitor); parser.Configuration = configuration; - var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess, configuration); + var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess, configuration.GetAllImplicitTokenClauses().Select(x => x.ImplicitToken).Distinct().ToList()); if (lexerResult.IsError) { foreach (var lexerResultError in lexerResult.Errors) diff --git a/sly/parser/generator/ExpressionRulesGenerator.cs b/sly/parser/generator/ExpressionRulesGenerator.cs index c6f81411..0c49a07f 100644 --- a/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/sly/parser/generator/ExpressionRulesGenerator.cs @@ -65,7 +65,7 @@ public ExpressionRulesGenerator(string i18n = null) bool isEnumValue = EnumConverter.IsEnumValue(attr.StringToken) || EnumConverter.IsEnumValue(attr.IntToken); OperationMetaData operation = null; - if (!isEnumValue && !string.IsNullOrEmpty(implicitToken) && implicitToken.StartsWith("'") && implicitToken.EndsWith("'")) // TODO ?? uniquement si contient '...' ?? + if (!isEnumValue && !string.IsNullOrEmpty(implicitToken) && implicitToken.StartsWith("'") && implicitToken.EndsWith("'")) { operation = new OperationMetaData(attr.Precedence, attr.Assoc, m, attr.Affix, implicitToken); } diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index a1d36235..9b25e324 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -76,13 +76,7 @@ public ParserBuilder() : this(null) var syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule); var visitor = new SyntaxTreeVisitor(configuration, parserInstance); parser = new Parser(I18n,syntaxParser, visitor); - var lexerResult = BuildLexer(extensionBuilder, lexerPostProcess); - parser.Lexer = lexerResult.Result; - if (lexerResult.IsError) - { - result.Errors.AddRange(lexerResult.Errors); - return result; - } + parser.Instance = parserInstance; parser.Configuration = configuration; result.Result = parser; @@ -107,8 +101,7 @@ public ParserBuilder() : this(null) result.Result.Configuration = expressionResult.Result; - // TODO : how to avoid this double lexer initialization ? - var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess, result.Result.Configuration); + var lexerResult = BuildLexer(extensionBuilder,lexerPostProcess, result.Result.Configuration.GetAllImplicitTokenClauses().Select(x => x.ImplicitToken).Distinct().ToList()); if (lexerResult.IsError) { foreach (var lexerResultError in lexerResult.Errors) @@ -188,44 +181,11 @@ public ParserBuilder() : this(null) protected virtual BuildResult> BuildLexer(BuildExtension extensionBuilder = null, - LexerPostProcess lexerPostProcess = null, ParserConfiguration parserConfiguration = null) + LexerPostProcess lexerPostProcess = null, IList implicitTokens = null) { - var lexer = LexerBuilder.BuildLexer(new BuildResult>(), extensionBuilder, I18n, lexerPostProcess); - if (parserConfiguration != null) - { - var implicitTokenClauses = parserConfiguration.GetAllImplicitTokenClauses(); - if (implicitTokenClauses.Any()) - { - Console.WriteLine($"found {implicitTokenClauses.Count} implicit tokens"); - var implicits = implicitTokenClauses.Select(x => x.ImplicitToken).Distinct(); - Console.WriteLine(string.Join(", ", implicits)); - if (lexer.IsOk && lexer.Result is GenericLexer genericLexer) - { - Console.WriteLine("something really magic will happen here !"); - foreach (var @implicit in implicits) - { - var x = genericLexer.FSMBuilder.Fsm.Run(@implicit, new LexerPosition()); - if (x.IsSuccess) - { - var t = genericLexer.FSMBuilder.Marks; - var y = genericLexer.FSMBuilder.Marks.FirstOrDefault(k => k.Value == x.NodeId); - if (y.Key == GenericLexer.in_identifier) // implicit keyword - { - var result = new BuildResult>(); - genericLexer.AddKeyWord(default(IN),@implicit,result); - ; - } - Console.WriteLine("hey ! Rodriguez !"); - } - else - { - var result = new BuildResult>(); - genericLexer.AddSugarLexem(default(IN),result,@implicit); - } - } - } - } - } + + + var lexer = LexerBuilder.BuildLexer(new BuildResult>(), extensionBuilder, I18n, lexerPostProcess, implicitTokens); return lexer; } From 9560bac421a889373526a01422071aaa91dcebd8 Mon Sep 17 00:00:00 2001 From: b3b00 Date: Wed, 8 Jun 2022 20:46:53 +0200 Subject: [PATCH 19/22] cleaning --- ParserTests/lexer/GenericLexerTests.cs | 10 +++- sly/lexer/TokenChannels.cs | 49 ------------------- sly/parser/generator/ParserConfiguration.cs | 2 +- .../visitor/GraphVizEBNFSyntaxTreeVisitor.cs | 24 ++++----- sly/parser/parser/GroupItem.cs | 2 + .../EBNFRecursiveDescentSyntaxParser.cs | 8 --- ...EBNFRecursiveDescentSyntaxParserStarter.cs | 19 ++----- sly/parser/syntax/tree/SyntaxEpsilon.cs | 6 --- 8 files changed, 27 insertions(+), 93 deletions(-) diff --git a/ParserTests/lexer/GenericLexerTests.cs b/ParserTests/lexer/GenericLexerTests.cs index db0a3c7c..5f1353d3 100644 --- a/ParserTests/lexer/GenericLexerTests.cs +++ b/ParserTests/lexer/GenericLexerTests.cs @@ -9,6 +9,7 @@ using sly.parser; using sly.parser.generator; using Xunit; +using Xunit.Abstractions; namespace ParserTests.lexer { @@ -467,6 +468,13 @@ public enum Issue177Regex public class GenericLexerTests { + private readonly ITestOutputHelper _testOutputHelper; + + public GenericLexerTests(ITestOutputHelper testOutputHelper) + { + _testOutputHelper = testOutputHelper; + } + [Fact] public void TestEmptyInput() { @@ -952,7 +960,7 @@ public void TestCharTokens() Assert.NotNull(lexer); var res1 = lexer.Tokenize("'c'"); Assert.False(res1.IsError); - Console.WriteLine(res1.Tokens.ToString()); + _testOutputHelper.WriteLine(res1.Tokens.ToString()); Assert.Equal(2, res1.Tokens.Count); Token token = res1.Tokens[0]; Assert.Equal('c', token.CharValue); diff --git a/sly/lexer/TokenChannels.cs b/sly/lexer/TokenChannels.cs index eabc45c2..7cd9615b 100644 --- a/sly/lexer/TokenChannels.cs +++ b/sly/lexer/TokenChannels.cs @@ -23,7 +23,6 @@ public class TokenChannels : IEnumerable> public List> Tokens => GetChannel(Channels.Main).Tokens.Where(x => x != null).ToList(); - public readonly int ChannelId; public TokenChannels() { @@ -54,11 +53,6 @@ public TokenChannel GetChannel(int i) { return _tokenChannels[i]; } - - public void SetChannel(int i, TokenChannel token) - { - _tokenChannels[i] = token; - } public Token this[int index] { @@ -112,48 +106,6 @@ public void Add(Token token) } - public bool ContainsChannel(int channel) => _tokenChannels.ContainsKey(channel); - - - - public Token TokenAt(int index) - { - foreach (var channel in _tokenChannels) - { - var token = TokenInChannelAt(channel.Value, index); - if (token != null) - { - return null; - } - } - return null; - } - - public Token TokenInChannelAt(TokenChannel channel, int index) - { - - if (channel != null) - { - if (index >= 0 && index < channel.Count) - { - return channel[index]; - } - } - return null; - } - - public Token TokenInChannelAt(int channelId, int index) - { - TokenChannel channel = null; - if (TryGet(index, out channel)) - { - if (index >= 0 && index < channel.Count) - { - return channel[index]; - } - } - return null; - } public bool TryGet(int index, out TokenChannel channel) => _tokenChannels.TryGetValue(index, out channel); @@ -169,7 +121,6 @@ IEnumerator IEnumerable.GetEnumerator() public override string ToString() { - var channels = _tokenChannels.Values.OrderBy(x => x.ChannelId); return string.Join("\n", _tokenChannels.Values.Select(x => x.ToString()).ToArray()); } } diff --git a/sly/parser/generator/ParserConfiguration.cs b/sly/parser/generator/ParserConfiguration.cs index 1503bc5d..88608c8a 100644 --- a/sly/parser/generator/ParserConfiguration.cs +++ b/sly/parser/generator/ParserConfiguration.cs @@ -48,7 +48,7 @@ public List> GetAllImplicitTokenClauses() if (clause is OptionClause option) { if (option.Clause is TerminalClause terminal && terminal.IsImplicitToken) - clauses.Add(terminal); + clauses.Add(terminal); } } } diff --git a/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs b/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs index ef418961..ebaba2f9 100644 --- a/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs +++ b/sly/parser/generator/visitor/GraphVizEBNFSyntaxTreeVisitor.cs @@ -142,20 +142,20 @@ private DotNode Visit(SyntaxNode node) - result = Node(GetNodeLabel(node)); - Graph.Add(result); - children.ForEach(c => + result = Node(GetNodeLabel(node)); + Graph.Add(result); + children.ForEach(c => + { + if (c != null) // Prevent arrows with null destinations { - if (c != null) // Prevent arrows with null destinations + var edge = new DotArrow(result, c) { - var edge = new DotArrow(result, c) - { - // Set all available properties - ArrowHeadShape = "none" - }; - Graph.Add(edge); - } - }); + // Set all available properties + ArrowHeadShape = "none" + }; + Graph.Add(edge); + } + }); return result; } diff --git a/sly/parser/parser/GroupItem.cs b/sly/parser/parser/GroupItem.cs index 1c4f2d11..8588d2dc 100644 --- a/sly/parser/parser/GroupItem.cs +++ b/sly/parser/parser/GroupItem.cs @@ -39,11 +39,13 @@ public X Match(Func, X> fToken, Func fValue return fValue(Name, Value); } + [ExcludeFromCodeCoverage] public static implicit operator OUT(GroupItem item) { return item.Match((name, token) => default(OUT), (name, value) => item.Value); } + [ExcludeFromCodeCoverage] public static implicit operator Token(GroupItem item) { return item.Match>((name, token) => item.Token, (name, value) => default(Token)); diff --git a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs index d948b0d6..fdecc415 100644 --- a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs +++ b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParser.cs @@ -260,15 +260,7 @@ public EBNFRecursiveDescentSyntaxParser(ParserConfiguration configurati return finalResult; } } - - } - else - { - throw new ParserConfigurationException( - $@"expression rule {rule.RuleString} is incorrect : must have ""nonterminal terminal nonterminal"" scheme"); } - - } var result = new SyntaxParseResult(); diff --git a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParserStarter.cs b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParserStarter.cs index 7ec39f68..c365a8dd 100644 --- a/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParserStarter.cs +++ b/sly/parser/parser/llparser/EBNFRecursiveDescentSyntaxParserStarter.cs @@ -15,19 +15,6 @@ public partial class EBNFRecursiveDescentSyntaxParser where IN : struct { switch (first) { - case TerminalClause clause: - { - var term = clause; - - InitStartingTokensWithTerminal(rule, term); - break; - } - case NonTerminalClause clause: - { - var nonterm = clause; - InitStartingTokensWithNonTerminal(rule, nonterm, nonTerminals); - break; - } case ZeroOrMoreClause zeroOrMore: { InitStartingTokensWithZeroOrMore(rule, zeroOrMore, nonTerminals); @@ -44,10 +31,10 @@ public partial class EBNFRecursiveDescentSyntaxParser where IN : struct rule.PossibleLeadingTokens.Add(terminalClause.ExpectedToken); break; } - case NonTerminalClause terminalClause: + case NonTerminalClause nonTerminalClause: { - InitStartingTokensForNonTerminal(nonTerminals, terminalClause.NonTerminalName); - NonTerminal nonTerminal = nonTerminals[terminalClause.NonTerminalName]; + InitStartingTokensForNonTerminal(nonTerminals, nonTerminalClause.NonTerminalName); + NonTerminal nonTerminal = nonTerminals[nonTerminalClause.NonTerminalName]; { rule.PossibleLeadingTokens.AddRange(nonTerminal.PossibleLeadingTokens); } diff --git a/sly/parser/syntax/tree/SyntaxEpsilon.cs b/sly/parser/syntax/tree/SyntaxEpsilon.cs index 2b453acc..ad6ccb72 100644 --- a/sly/parser/syntax/tree/SyntaxEpsilon.cs +++ b/sly/parser/syntax/tree/SyntaxEpsilon.cs @@ -4,12 +4,6 @@ namespace sly.parser.syntax.tree { public class SyntaxEpsilon : ISyntaxNode where IN : struct { - public SyntaxEpsilon() - { - - } - - public bool Discarded { get; } = false; public string Name => "Epsilon"; From 8090b32d6508f050b75305743eed21a737b4c053 Mon Sep 17 00:00:00 2001 From: Olivier Duhart <1224790+b3b00@users.noreply.github.com> Date: Wed, 8 Jun 2022 20:58:15 +0200 Subject: [PATCH 20/22] fix --- ParserTests/lexer/GenericLexerTests.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ParserTests/lexer/GenericLexerTests.cs b/ParserTests/lexer/GenericLexerTests.cs index 5f1353d3..0d92546e 100644 --- a/ParserTests/lexer/GenericLexerTests.cs +++ b/ParserTests/lexer/GenericLexerTests.cs @@ -468,11 +468,9 @@ public enum Issue177Regex public class GenericLexerTests { - private readonly ITestOutputHelper _testOutputHelper; - - public GenericLexerTests(ITestOutputHelper testOutputHelper) + + public GenericLexerTests() { - _testOutputHelper = testOutputHelper; } [Fact] @@ -960,7 +958,7 @@ public void TestCharTokens() Assert.NotNull(lexer); var res1 = lexer.Tokenize("'c'"); Assert.False(res1.IsError); - _testOutputHelper.WriteLine(res1.Tokens.ToString()); + Console.WriteLine(res1.Tokens.ToString()); Assert.Equal(2, res1.Tokens.Count); Token token = res1.Tokens[0]; Assert.Equal('c', token.CharValue); From a1f2926cda5893eb787680823ce5141f6b7a2d69 Mon Sep 17 00:00:00 2001 From: Olivier Duhart <1224790+b3b00@users.noreply.github.com> Date: Wed, 8 Jun 2022 21:28:19 +0200 Subject: [PATCH 21/22] Update SyntaxEpsilon.cs --- sly/parser/syntax/tree/SyntaxEpsilon.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sly/parser/syntax/tree/SyntaxEpsilon.cs b/sly/parser/syntax/tree/SyntaxEpsilon.cs index ad6ccb72..d3c412ff 100644 --- a/sly/parser/syntax/tree/SyntaxEpsilon.cs +++ b/sly/parser/syntax/tree/SyntaxEpsilon.cs @@ -9,14 +9,16 @@ public class SyntaxEpsilon : ISyntaxNode where IN : struct public bool HasByPassNodes { get; set; } = false; + [ExcludeFromCodeCoverage] public string Dump(string tab) { return $"Epsilon"; } + [ExcludeFromCodeCoverage] public string ToJson(int index = 0) { return $@"""{index}.Epsilon"":""e"""; } } -} \ No newline at end of file +} From 70362f2c057363cdd5fb157bc2ab74fe4c45981e Mon Sep 17 00:00:00 2001 From: Olivier Duhart <1224790+b3b00@users.noreply.github.com> Date: Wed, 8 Jun 2022 21:32:03 +0200 Subject: [PATCH 22/22] Update SyntaxEpsilon.cs --- sly/parser/syntax/tree/SyntaxEpsilon.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/sly/parser/syntax/tree/SyntaxEpsilon.cs b/sly/parser/syntax/tree/SyntaxEpsilon.cs index d3c412ff..cdabf91a 100644 --- a/sly/parser/syntax/tree/SyntaxEpsilon.cs +++ b/sly/parser/syntax/tree/SyntaxEpsilon.cs @@ -1,4 +1,5 @@ using sly.lexer; +using System.Diagnostics.CodeAnalysis; namespace sly.parser.syntax.tree {