Skip to content

Commit

Permalink
Merge 38cf139 into 545a022
Browse files Browse the repository at this point in the history
  • Loading branch information
@b3b00
b3b00 committed May 7, 2024
2 parents 545a022 + 38cf139 commit 38d4fac
Show file tree
Hide file tree
Showing 7 changed files with 134 additions and 9 deletions.
34 changes: 34 additions & 0 deletions Tests/Tests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,4 +258,38 @@ public void TestDateToken()
Check.That(dateTime).IsEqualTo("2024.04.23");
}

[Fact]
public void TestLexerOptions()
{
var grammar = @"
genericLexer MinimalLexer;
[IndentationAware(false)]
[IgnoreKeyWordCase(true)]
[AlphaNumDashId] ID;
[KeyWord] HELLO : ""hello"";
[KeyWord] WORLD : ""world"";
parser MinimalParser;
-> root : HELLO WORLD ;
";
var builder = new ParserBuilder();
var model = builder.CompileModel(grammar, "MinimalParser");
Check.That(model).IsOkModel();
Check.That(model.Value.LexerModel.Options.IgnoreKeyWordCase.Value).IsTrue();
Check.That(model.Value.LexerModel.Options.IndentationAware.Value).IsFalse();
Check.That(model.Value.LexerModel.Options.IgnoreWS).IsNull();
Check.That(model.Value.LexerModel.Options.IgnoreEOL).IsNull();

var generator = new LexerGenerator();
var lexer = generator.GenerateLexer(model.Value.LexerModel, "namespace");
;
var json = builder.Getz(grammar, "hello world", "MyParser", new List<(string format, SyntaxTreeProcessor processor)>() {("JSON",ParserBuilder.SyntaxTreeToJson)});
Check.That(json.IsError).IsFalse();
json = builder.Getz(grammar, "HELLO woRld", "MyParser", new List<(string format, SyntaxTreeProcessor processor)>() {("JSON",ParserBuilder.SyntaxTreeToJson)});
Check.That(json.IsError).IsFalse();

}

}
29 changes: 29 additions & 0 deletions csly-cli-builder/LexerBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Reflection.Emit;
using System.Text.Json.Serialization;
using clsy.cli.builder.parser.cli.model;
using clsy.cli.model.lexer;
using csly.cli.model.lexer;
using sly.lexer;
using sly.lexer.fsm;
Expand Down Expand Up @@ -35,6 +36,8 @@ public LexerBuilder(string name)

EnumBuilder enumBuilder = moduleBuilder.DefineEnum(DynamicLexerName, TypeAttributes.Public, typeof(int));

SetLexerOptions(enumBuilder, model.Options);

int i = 0;

enumBuilder.DefineLiteral($"{model.Name.ToUpper()}_EOS", i);
Expand All @@ -57,6 +60,32 @@ public LexerBuilder(string name)
return (finished,extensionBuilder,dynamicAssembly,moduleBuilder);
}


private void SetLexerOptions(EnumBuilder builder, LexerOptions options)
{
var attributeType = typeof(LexerAttribute);
ConstructorInfo constructorInfo = attributeType.GetConstructor(
new Type[3] { typeof(string), typeof(bool), typeof(int) });

CustomAttributeBuilder customAttributeBuilder = new CustomAttributeBuilder(
attributeType.GetConstructor(Type.EmptyTypes),
new object[0],
new PropertyInfo?[] { // properties to assign to
attributeType.GetProperty(nameof(LexerAttribute.IgnoreEOL)),
attributeType.GetProperty(nameof(LexerAttribute.IgnoreWS)),
attributeType.GetProperty(nameof(LexerAttribute.KeyWordIgnoreCase)),
attributeType.GetProperty(nameof(LexerAttribute.IndentationAWare)),
}!,
new object[] { // values for property assignment
options.IgnoreEOL ?? true,
options.IgnoreWS ?? true,
options.IgnoreKeyWordCase ?? false,
options.IndentationAware ?? false
});

builder.SetCustomAttribute(customAttributeBuilder);
}

private Delegate BuildExtensionIfNeeded(LexerModel model, Type? enumType)
{

Expand Down
7 changes: 6 additions & 1 deletion csly-cli-model/lexer/LexerModel.cs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
using clsy.cli.model.lexer;
using csly.cli.model;
using csly.cli.model.lexer;
using csly.cli.model.parser;
using sly.lexer;

namespace clsy.cli.builder.parser.cli.model;

public class LexerModel : ICLIModel
{

public LexerOptions Options { get; set; }
public List<TokenModel> Tokens { get; set; }

public Dictionary<string, List<TokenModel>> TokensByName { get; set; }
Expand All @@ -14,10 +18,11 @@ public class LexerModel : ICLIModel

public bool HasExtension => Tokens.Exists(x => x.Type == GenericToken.Extension);

public LexerModel(List<TokenModel> tokens, string name)
public LexerModel(List<TokenModel> tokens, LexerOptions options, string name)
{
Name = name;
Tokens = tokens;
Options = options;
var grouped = tokens.GroupBy(x => x.Name);
TokensByName = grouped.ToDictionary(x => x.Key, x => x.ToList());
}
Expand Down
14 changes: 14 additions & 0 deletions csly-cli-model/lexer/LexerOptions.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
using csly.cli.model;

namespace clsy.cli.model.lexer;

public class LexerOptions : ICLIModel
{
public bool? IgnoreWS { get; set; }

public bool? IgnoreEOL { get; set; }

public bool? IgnoreKeyWordCase { get; set; }

public bool? IndentationAware { get; set; }
}
2 changes: 1 addition & 1 deletion csly-cli-model/parser/Optimization.cs → csly-cli-model/parser/ParserOptimization.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

namespace csly.cli.model.parser;

public class Optimization : ICLIModel
public class ParserOptimization : ICLIModel
{
public bool UseMemoization { get; set; }
public bool BroadenTokenWindow { get; set; }
Expand Down
46 changes: 39 additions & 7 deletions csly-cli-parser/CLIParser.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using clsy.cli.builder.parser.cli.model;
using clsy.cli.model.lexer;
using csly.cli.model;
using csly.cli.model.lexer;
using csly.cli.model.parser;
Expand All @@ -23,10 +24,10 @@ public ICLIModel Root(ICLIModel genericLex, ICLIModel parser, ParserContext cont
return new Model(genericLex as LexerModel, parser as ParserModel) ;
}

[Production("parserRoot : PARSER[d] ID SEMICOLON[d] optimization* rule*")]
[Production("parserRoot : PARSER[d] ID SEMICOLON[d] parser_optimization* rule*")]
public ICLIModel Parser(Token<CLIToken> name, List<ICLIModel> optimizations, List<ICLIModel> rules, ParserContext context)
{
var optims = optimizations.Cast<Optimization>().ToList();
var optims = optimizations.Cast<ParserOptimization>().ToList();
var model = new ParserModel()
{
UseMemoization = optims.Exists(x => x.UseMemoization),
Expand All @@ -40,10 +41,10 @@ public ICLIModel Parser(Token<CLIToken> name, List<ICLIModel> optimizations, Lis

}

[Production("optimization : LEFTBRACKET[d] [USEMEMOIZATION|BROADENTOKENWINDOW] RIGHTBRACKET[d]")]
[Production("parser_optimization : LEFTBRACKET[d] [USEMEMOIZATION|BROADENTOKENWINDOW] RIGHTBRACKET[d]")]
public ICLIModel Optimization(Token<CLIToken> optimizationToken, ParserContext context)
{
return new Optimization()
return new ParserOptimization()
{
UseMemoization = optimizationToken.TokenID == CLIToken.USEMEMOIZATION,
BroadenTokenWindow = optimizationToken.TokenID == CLIToken.BROADENTOKENWINDOW
Expand All @@ -56,10 +57,19 @@ public ICLIModel Optimization(Token<CLIToken> optimizationToken, ParserContext c

#region generic lexer

[Production("genericRoot : GENERICLEXER[d] ID SEMICOLON[d] modedToken*")]
public ICLIModel Lexer(Token<CLIToken> name, List<ICLIModel> tokens, ParserContext context)
[Production("genericRoot : GENERICLEXER[d] ID SEMICOLON[d] lexer_option* modedToken*")]
public ICLIModel Lexer(Token<CLIToken> name, List<ICLIModel> optionList, List<ICLIModel> tokens, ParserContext context)
{
return new LexerModel(tokens.Cast<TokenModel>().ToList(), name.Value);
var opts = optionList.Cast<LexerOptions>();
var options = new LexerOptions()
{

IgnoreWS = opts.Select(x => x.IgnoreWS).FirstOrDefault(x => x.HasValue),
IndentationAware = opts.Select(x => x.IndentationAware).FirstOrDefault(x => x.HasValue),
IgnoreEOL = opts.Select(x => x.IgnoreEOL).FirstOrDefault(x => x.HasValue),
IgnoreKeyWordCase = opts.Select(x => x.IgnoreKeyWordCase).FirstOrDefault(x => x.HasValue)
};
return new LexerModel(tokens.Cast<TokenModel>().ToList(),options, name.Value);
}

[Production("modedToken : mode* token")]
Expand Down Expand Up @@ -304,7 +314,29 @@ public ICLIModel RangeDefinition(Token<CLIToken> start, Token<CLIToken> end, Par

#endregion

#region lexer options

// [Keyword("IndentationAware")] INDENTATIONAWARE,
// [Keyword("IgnoreWhiteSpaces")] IGNOREWHITESPACES,
// [Keyword("IgnoreEndOfLines")] IGNOREEOL,
// [Keyword("IgnoreKeyWordCase")] IGNOREKEYWORDCASING,

[Production(
"lexer_option : LEFTBRACKET[d] [IGNOREKEYWORDCASING|INDENTATIONAWARE|IGNOREWHITESPACES|IGNOREEOL] LEFTPAREN[d][TRUE|FALSE]RIGHTPAREN[d] RIGHTBRACKET[d]")]
public ICLIModel lexerOption(Token<CLIToken> option, Token<CLIToken> enabledFlag, ParserContext context)
{
bool enabled = enabledFlag.Value == "true";
return new LexerOptions()
{
IgnoreWS = option.TokenID == CLIToken.IGNOREWHITESPACES ? enabled : null,
IgnoreEOL = option.TokenID == CLIToken.IGNOREEOL ? enabled : null,
IgnoreKeyWordCase = option.TokenID == CLIToken.IGNOREKEYWORDCASING ? enabled : null,
IndentationAware = option.TokenID == CLIToken.INDENTATIONAWARE ? enabled : null
};
}


#endregion

#region parser

Expand Down
11 changes: 11 additions & 0 deletions csly-cli-parser/CLIToken.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,20 @@ public enum CLIToken
[Keyword("Push")] PUSH,
[Keyword("Mode")] MODE,
[Keyword("Pop")] POP,
[Keyword("true")] TRUE,
[Keyword("false")] FALSE,


// parser optimizations
[Keyword("UseMemoization")] USEMEMOIZATION,
[Keyword("BroadenTokenWindow")] BROADENTOKENWINDOW,

// lexer options
[Keyword("IndentationAware")] INDENTATIONAWARE,
[Keyword("IgnoreWhiteSpaces")] IGNOREWHITESPACES,
[Keyword("IgnoreEndOfLines")] IGNOREEOL,
[Keyword("IgnoreKeyWordCase")] IGNOREKEYWORDCASING,

[Keyword("YYYYMMDD")] YYYYMMDD,
[Keyword("DDMMYYYY")] DDMMYYYY,

Expand Down

0 comments on commit 38d4fac

Please sign in to comment.