Skip to content

Experiment/optimizations #551

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 27, 2025
Merged

Experiment/optimizations #551

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d1c811e
optimization : use hashset to dedup errors
b3b00 Mar 27, 2025
f437bba
limit GetErrors() calls
b3b00 Mar 27, 2025
5c48ed8
use array instead of List
b3b00 Mar 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/sly/parser/parser/ISyntaxParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public interface ISyntaxParser<IN, OUT> where IN : struct

Dictionary<IN, Dictionary<string, string>> LexemeLabels { get; set; }

SyntaxParseResult<IN, OUT> Parse(IList<Token<IN>> tokens, string startingNonTerminal = null);
SyntaxParseResult<IN, OUT> Parse(Token<IN>[] tokens, string startingNonTerminal = null);

void Init(ParserConfiguration<IN, OUT> configuration, string root);

Expand Down
2 changes: 1 addition & 1 deletion src/sly/parser/parser/Parser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public ParseResult<IN, OUT> ParseWithContext(IList<Token<IN>> tokens, object par
var result = new ParseResult<IN, OUT>();

var cleaner = new SyntaxTreeCleaner<IN, OUT>();
var syntaxResult = SyntaxParser.Parse(tokens, startingNonTerminal);
var syntaxResult = SyntaxParser.Parse(tokens.ToArray(), startingNonTerminal);
syntaxResult.UsesOperations = Configuration.UsesOperations;
syntaxResult = cleaner.CleanSyntaxTree(syntaxResult);
if (!syntaxResult.IsError && syntaxResult.Root != null)
Expand Down
16 changes: 7 additions & 9 deletions src/sly/parser/parser/SyntaxParseResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using sly.parser.syntax.grammar;
using sly.parser.syntax.tree;

Expand All @@ -14,16 +15,17 @@ public class SyntaxParseResult<IN, OUT> where IN : struct



private List<UnexpectedTokenSyntaxError<IN>> Errors { get; set; }
private HashSet<UnexpectedTokenSyntaxError<IN>> Errors { get; set; }

public int EndingPosition { get; set; }

public bool IsEnded { get; set; }

private void InitErrors()
{
if (Errors == null) {
Errors = new List<UnexpectedTokenSyntaxError<IN>>();
if (Errors == null)
{
Errors = new HashSet<UnexpectedTokenSyntaxError<IN>>();
}
}

Expand All @@ -39,14 +41,10 @@ public void AddErrors(IList<UnexpectedTokenSyntaxError<IN>> errors)
public void AddError(UnexpectedTokenSyntaxError<IN> error)
{
InitErrors();
if (!Errors.Contains(error))
{
// #493 : dedup errors
Errors.Add(error);
}
Errors.Add(error);
}

public IList<UnexpectedTokenSyntaxError<IN>> GetErrors() => Errors;
public IList<UnexpectedTokenSyntaxError<IN>> GetErrors() => Errors?.ToList();

public List<LeadingToken<IN>> Expecting {get; set;}

Expand Down
24 changes: 13 additions & 11 deletions src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ public partial class RecursiveDescentSyntaxParser<IN, OUT> where IN : struct
{
#region parsing

public SyntaxParseResult<IN, OUT> ParseNonTerminal(IList<Token<IN>> tokens, NonTerminalClause<IN, OUT> nonTermClause,
public SyntaxParseResult<IN, OUT> ParseNonTerminal(Token<IN>[] tokens, NonTerminalClause<IN, OUT> nonTermClause,
int currentPosition, SyntaxParsingContext<IN, OUT> parsingContext)
{
var result = ParseNonTerminal(tokens, nonTermClause.NonTerminalName, currentPosition, parsingContext);
return result;
}

public SyntaxParseResult<IN, OUT> ParseNonTerminal(IList<Token<IN>> tokens, string nonTerminalName,
public SyntaxParseResult<IN, OUT> ParseNonTerminal(Token<IN>[] tokens, string nonTerminalName,
int currentPosition, SyntaxParsingContext<IN, OUT> parsingContext)
{
if (parsingContext.TryGetParseResult(new NonTerminalClause<IN, OUT>(nonTerminalName), currentPosition,
Expand All @@ -38,24 +38,25 @@ public SyntaxParseResult<IN, OUT> ParseNonTerminal(IList<Token<IN>> tokens, stri
while (i < rules.Count)
{
var innerrule = rules[i];
if (startPosition < tokens.Count
if (startPosition < tokens.Length
&& (!tokens[startPosition].IsEOS || (tokens[startPosition].IsEOS && innerrule.MayBeEmpty))
&& innerrule.Match(tokens, startPosition, Configuration))
{
var innerRuleRes = Parse(tokens, innerrule, startPosition, nonTerminalName, parsingContext);
rulesResults.Add(innerRuleRes);

var other = greaterIndex == 0 && innerRuleRes.EndingPosition == 0;
if (innerRuleRes.EndingPosition > greaterIndex && innerRuleRes.GetErrors() != null &&
innerRuleRes.GetErrors().Count == 0 || other)
var innerRuleResErrors = innerRuleRes.GetErrors();
if (innerRuleRes.EndingPosition > greaterIndex && innerRuleResErrors != null &&
innerRuleResErrors.Count == 0 || other)
{
greaterIndex = innerRuleRes.EndingPosition;
if (innerRuleRes.GetErrors() != null)
innerRuleErrors.AddRange(innerRuleRes.GetErrors());
if (innerRuleResErrors != null)
innerRuleErrors.AddRange(innerRuleResErrors);
}

if (innerRuleRes.GetErrors() != null)
innerRuleErrors.AddRange(innerRuleRes.GetErrors());
if (innerRuleResErrors != null)
innerRuleErrors.AddRange(innerRuleResErrors);
}

i++;
Expand Down Expand Up @@ -123,9 +124,10 @@ public SyntaxParseResult<IN, OUT> ParseNonTerminal(IList<Token<IN>> tokens, stri
List<UnexpectedTokenSyntaxError<IN>> terr = new List<UnexpectedTokenSyntaxError<IN>>();
foreach (var ruleResult in rulesResults)
{
if (ruleResult.GetErrors() != null)
var ruleErrors = ruleResult.GetErrors();
if (ruleErrors != null)
{
terr.AddRange(ruleResult.GetErrors());
terr.AddRange(ruleErrors);
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Terminal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ public partial class RecursiveDescentSyntaxParser<IN, OUT> where IN : struct
{
#region parsing

public SyntaxParseResult<IN, OUT> ParseTerminal(IList<Token<IN>> tokens, TerminalClause<IN, OUT> terminal, int position,
public SyntaxParseResult<IN, OUT> ParseTerminal(Token<IN>[] tokens, TerminalClause<IN, OUT> terminal, int position,
SyntaxParsingContext<IN, OUT> parsingContext)
{
if (parsingContext.TryGetParseResult(terminal, position, out var parseResult))
Expand Down
19 changes: 10 additions & 9 deletions src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ public RecursiveDescentSyntaxParser(ParserConfiguration<IN, OUT> configuration,

#region parsing

public SyntaxParseResult<IN, OUT> Parse(IList<Token<IN>> tokens, string startingNonTerminal = null)
public SyntaxParseResult<IN, OUT> Parse(Token<IN>[] tokens, string startingNonTerminal = null)
{
return SafeParse(tokens, new SyntaxParsingContext<IN, OUT>(Configuration.UseMemoization), startingNonTerminal);
}

public SyntaxParseResult<IN, OUT> SafeParse(IList<Token<IN>> tokens, SyntaxParsingContext<IN, OUT> parsingContext, string startingNonTerminal = null)
public SyntaxParseResult<IN, OUT> SafeParse(Token<IN>[] tokens, SyntaxParsingContext<IN, OUT> parsingContext, string startingNonTerminal = null)
{
var start = startingNonTerminal ?? StartingNonTerminal;
var NonTerminals = Configuration.NonTerminals;
Expand Down Expand Up @@ -93,7 +93,7 @@ public SyntaxParseResult<IN, OUT> SafeParse(IList<Token<IN>> tokens, SyntaxParsi
}
else
{
if (result.EndingPosition < tokens.Count-1)
if (result.EndingPosition < tokens.Length-1)
{
SyntaxParseResult<IN, OUT> r = new SyntaxParseResult<IN, OUT>()
{
Expand Down Expand Up @@ -133,7 +133,7 @@ public SyntaxParseResult<IN, OUT> SafeParse(IList<Token<IN>> tokens, SyntaxParsi
}


public virtual SyntaxParseResult<IN, OUT> Parse(IList<Token<IN>> tokens, Rule<IN, OUT> rule, int position,
public virtual SyntaxParseResult<IN, OUT> Parse(Token<IN>[] tokens, Rule<IN, OUT> rule, int position,
string nonTerminalName, SyntaxParsingContext<IN, OUT> parsingContext)
{
var currentPosition = position;
Expand Down Expand Up @@ -169,16 +169,17 @@ public virtual SyntaxParseResult<IN, OUT> Parse(IList<Token<IN>> tokens, Rule<IN
{
var nonTerminalResult =
ParseNonTerminal(tokens, terminalClause, currentPosition, parsingContext);
var ntErrors = nonTerminalResult.GetErrors();
if (!nonTerminalResult.IsError)
{
children.Add(nonTerminalResult.Root);
currentPosition = nonTerminalResult.EndingPosition;
if (nonTerminalResult.GetErrors() != null && nonTerminalResult.GetErrors().Count > 0)
errors.AddRange(nonTerminalResult.GetErrors());
if (ntErrors != null && ntErrors.Count > 0)
errors.AddRange(ntErrors);
}
else
{
errors.AddRange(nonTerminalResult.GetErrors());
errors.AddRange(ntErrors);
}

isError = nonTerminalResult.IsError;
Expand Down Expand Up @@ -214,12 +215,12 @@ public virtual SyntaxParseResult<IN, OUT> Parse(IList<Token<IN>> tokens, Rule<IN



private SyntaxParseResult<IN, OUT> NoMatchingRuleError(IList<Token<IN>> tokens, int currentPosition,
private SyntaxParseResult<IN, OUT> NoMatchingRuleError(Token<IN>[] tokens, int currentPosition,
List<LeadingToken<IN>> allAcceptableTokens)
{
var noRuleErrors = new List<UnexpectedTokenSyntaxError<IN>>();

if (currentPosition < tokens.Count)
if (currentPosition < tokens.Length)
{
noRuleErrors.Add(new UnexpectedTokenSyntaxError<IN>(tokens[currentPosition], I18n,
allAcceptableTokens));
Expand Down
2 changes: 1 addition & 1 deletion ...llparser/ebnf/EBNFRecursiveDescentSyntaxParser.EBNFRecursiveDescentSyntaxParser.Choice.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public partial class EBNFRecursiveDescentSyntaxParser<IN, OUT> where IN : struct
{
#region parsing

public SyntaxParseResult<IN, OUT> ParseChoice(IList<Token<IN>> tokens, ChoiceClause<IN, OUT> clause,
public SyntaxParseResult<IN, OUT> ParseChoice(Token<IN>[] tokens, ChoiceClause<IN, OUT> clause,
int position, SyntaxParsingContext<IN, OUT> parsingContext)
{
if (parsingContext.TryGetParseResult(clause, position, out var parseResult))
Expand Down
14 changes: 7 additions & 7 deletions src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ public partial class EBNFRecursiveDescentSyntaxParser<IN, OUT>
{
#region parsing

public virtual SyntaxParseResult<IN, OUT> ParseInfixExpressionRule(IList<Token<IN>> tokens, Rule<IN, OUT> rule,
public virtual SyntaxParseResult<IN, OUT> ParseInfixExpressionRule(Token<IN>[] tokens, Rule<IN, OUT> rule,
int position,
string nonTerminalName, SyntaxParsingContext<IN, OUT> parsingContext)
{
Expand Down Expand Up @@ -97,9 +97,9 @@ public virtual SyntaxParseResult<IN, OUT> ParseInfixExpressionRule(IList<Token<I
finalNode = ManageExpressionRules(rule, finalNode);
var finalResult = new SyntaxParseResult<IN, OUT>();
finalResult.Root = finalNode;
finalResult.IsEnded = currentPosition >= tokens.Count - 1
|| currentPosition == tokens.Count - 2 &&
tokens[tokens.Count - 1].IsEOS;
finalResult.IsEnded = currentPosition >= tokens.Length - 1
|| currentPosition == tokens.Length - 2 &&
tokens[tokens.Length - 1].IsEOS;
finalResult.EndingPosition = currentPosition;
return finalResult;
}
Expand All @@ -120,9 +120,9 @@ public virtual SyntaxParseResult<IN, OUT> ParseInfixExpressionRule(IList<Token<I
if (node.IsByPassNode) // inutile de créer un niveau supplémentaire
result.Root = children[0];
result.Root = node;
result.IsEnded = result.EndingPosition >= tokens.Count - 1
|| result.EndingPosition == tokens.Count - 2 &&
tokens[tokens.Count - 1].IsEOS;
result.IsEnded = result.EndingPosition >= tokens.Length - 1
|| result.EndingPosition == tokens.Length - 2 &&
tokens[tokens.Length - 1].IsEOS;
return result;
}

Expand Down
15 changes: 8 additions & 7 deletions src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ public partial class EBNFRecursiveDescentSyntaxParser<IN, OUT> where IN : struct
{
#region parsing

public SyntaxParseResult<IN, OUT> ParseZeroOrMore(IList<Token<IN>> tokens, ZeroOrMoreClause<IN, OUT> clause, int position,
public SyntaxParseResult<IN, OUT> ParseZeroOrMore(Token<IN>[] tokens, ZeroOrMoreClause<IN, OUT> clause, int position,
SyntaxParsingContext<IN, OUT> parsingContext)
{
if (parsingContext.TryGetParseResult(clause, position, out var parseResult))
Expand Down Expand Up @@ -67,9 +67,10 @@ public SyntaxParseResult<IN, OUT> ParseZeroOrMore(IList<Token<IN>> tokens, ZeroO
currentPosition = innerResult.EndingPosition;
lastInnerResult = innerResult;
hasByPasNodes = hasByPasNodes || innerResult.HasByPassNodes;
if (lastInnerResult.GetErrors() != null)
var lastInnerErrors = lastInnerResult.GetErrors();
if (lastInnerErrors != null)
{
innerErrors.AddRange(lastInnerResult.GetErrors());
innerErrors.AddRange(lastInnerErrors);
}
}
else
Expand All @@ -80,7 +81,7 @@ public SyntaxParseResult<IN, OUT> ParseZeroOrMore(IList<Token<IN>> tokens, ZeroO
}
}

stillOk = innerResult != null && !innerResult.IsError && currentPosition < tokens.Count;
stillOk = innerResult != null && !innerResult.IsError && currentPosition < tokens.Length;
}


Expand All @@ -94,7 +95,7 @@ public SyntaxParseResult<IN, OUT> ParseZeroOrMore(IList<Token<IN>> tokens, ZeroO
return result;
}

public SyntaxParseResult<IN, OUT> ParseRepeat(IList<Token<IN>> tokens, RepeatClause<IN, OUT> clause, int position,
public SyntaxParseResult<IN, OUT> ParseRepeat(Token<IN>[] tokens, RepeatClause<IN, OUT> clause, int position,
SyntaxParsingContext<IN, OUT> parsingContext)
{
if (parsingContext.TryGetParseResult(clause, position, out var parseResult))
Expand Down Expand Up @@ -155,7 +156,7 @@ public SyntaxParseResult<IN, OUT> ParseRepeat(IList<Token<IN>> tokens, RepeatCla
return result;
}

private SyntaxParseResult<IN, OUT> ParseInnerRepeat(IList<Token<IN>> tokens, SyntaxParsingContext<IN, OUT> parsingContext, IClause<IN, OUT> innerClause,
private SyntaxParseResult<IN, OUT> ParseInnerRepeat(Token<IN>[] tokens, SyntaxParsingContext<IN, OUT> parsingContext, IClause<IN, OUT> innerClause,
ManySyntaxNode<IN, OUT> manyNode, int currentPosition, out bool hasByPasNodes)
{
SyntaxParseResult<IN, OUT> innerResult;
Expand Down Expand Up @@ -189,7 +190,7 @@ private SyntaxParseResult<IN, OUT> ParseInnerRepeat(IList<Token<IN>> tokens, Syn
return innerResult;
}

public SyntaxParseResult<IN, OUT> ParseOneOrMore(IList<Token<IN>> tokens, OneOrMoreClause<IN, OUT> clause, int position,
public SyntaxParseResult<IN, OUT> ParseOneOrMore(Token<IN>[] tokens, OneOrMoreClause<IN, OUT> clause, int position,
SyntaxParsingContext<IN, OUT> parsingContext)
{
if (parsingContext.TryGetParseResult(clause, position, out var parseResult))
Expand Down
2 changes: 1 addition & 1 deletion src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public partial class EBNFRecursiveDescentSyntaxParser<IN, OUT> where IN : struct
{
#region parsing

public SyntaxParseResult<IN, OUT> ParseOption(IList<Token<IN>> tokens, OptionClause<IN, OUT> clause, Rule<IN, OUT> rule,
public SyntaxParseResult<IN, OUT> ParseOption(Token<IN>[] tokens, OptionClause<IN, OUT> clause, Rule<IN, OUT> rule,
int position, SyntaxParsingContext<IN, OUT> parsingContext)
{
if (parsingContext.TryGetParseResult(clause, position, out var parseResult))
Expand Down
8 changes: 4 additions & 4 deletions src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public EBNFRecursiveDescentSyntaxParser(ParserConfiguration<IN, OUT> configurati

#region parsing

public override SyntaxParseResult<IN, OUT> Parse(IList<Token<IN>> tokens, Rule<IN, OUT> rule, int position,
public override SyntaxParseResult<IN, OUT> Parse(Token<IN>[] tokens, Rule<IN, OUT> rule, int position,
string nonTerminalName, SyntaxParsingContext<IN, OUT> parsingContext)
{
if (rule.IsInfixExpressionRule && rule.IsExpressionRule)
Expand Down Expand Up @@ -147,9 +147,9 @@ public override SyntaxParseResult<IN, OUT> Parse(IList<Token<IN>> tokens, Rule<I
node = new GroupSyntaxNode<IN, OUT>(nonTerminalName, children);
node = ManageExpressionRules(rule, node);
result.Root = node;
result.IsEnded = currentPosition >= tokens.Count - 1
|| currentPosition == tokens.Count - 2 &&
tokens[tokens.Count - 1].IsEOS;
result.IsEnded = currentPosition >= tokens.Length - 1
|| currentPosition == tokens.Length - 2 &&
tokens[tokens.Length - 1].IsEOS;
}
else
{
Expand Down
Loading