fix keyword token accepts many definitions #26

b3b00 · Jun 6, 2024 · d752900 · d752900
1 parent 3d099b1
commit d752900
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 3 deletions.
diff --git a/Tester/Program.cs b/Tester/Program.cs
@@ -28,7 +28,8 @@ public static List<string> GetLines(this string content)
     public static void Main(string[] args)
     {
         //Compile("C:\\Users\\olduh\\dev\\BlazorCslyViz\\BlazorVizView\\samples\\grammar\\indented-while.txt");
-        Extract(@"C:\Users\olduh\dev\csly\src\samples\XML\MinimalXmlParser.cs","C:\\Users\\olduh\\dev\\csly\\src\\samples\\XML\\MinimalXmlLexer.cs", @"C:\Users\olduh\dev\BlazorCslyViz\BlazorVizView\samples\grammar\xml.txt");
+        // Extract(@"C:\Users\olduh\dev\csly\src\samples\jsonparser\EbnfJsonGenericParser.cs","C:\\Users\\olduh\\dev\\csly\\src\\samples\\jsonparser\\JsonTokenGeneric.cs", @"C:\Users\olduh\dev\BlazorCslyViz\BlazorVizView\samples\grammar\json.txt");
+        Parse(@"C:\Users\olduh\dev\BlazorCslyViz\BlazorVizView\samples\grammar\json.txt", @"C:\Users\olduh\dev\BlazorCslyViz\BlazorVizView\samples\source\json.txt");
     }
 
 
@@ -128,5 +129,28 @@ private static void Extract(string parserPath, string lexrerPath, string outputP
         }
 
     }
+
+    private static void Parse(string grammarPath, string sourcePath, string outputPath = null)
+        {
+            var e = CslyProcessor.GetDot(File.ReadAllText(grammarPath), File.ReadAllText(sourcePath));
+            if (e.IsOK)
+            {
+                if (outputPath != null)
+                {
+                    File.WriteAllText(outputPath, e.Result);
+                }
+
+                Console.WriteLine("parse ok");
+
+            }
+            else
+            {
+                foreach (var error in e.Errors)
+                {
+                    Console.WriteLine(error);
+                }
+            }
+
+        }
 }
 }
diff --git a/csly-cli-extractor/LexerSpecificationExtractor.cs b/csly-cli-extractor/LexerSpecificationExtractor.cs
@@ -96,7 +96,9 @@ private string Lexeme(string name, GenericToken type, params string[] args)
             }
             case GenericToken.KeyWord:
             {
-                return $"[KeyWord] {name} : \"{args[0].Replace("\"","\\\"")}\";";
+                string definitions = string.Join(" ", args.Select(x => "\"" + x.Replace("\"", "\\\"") + "\""));
+
+                return $"[KeyWord] {name} : {definitions};";
             }
             case GenericToken.SugarToken:
             {

diff --git a/csly-cli-parser/CLIParser.cs b/csly-cli-parser/CLIParser.cs
@@ -132,7 +132,7 @@ public ICLIModel ModeDefault(ParserContext context)
 
 
     [Production(
-        "token : attribute* LEFTBRACKET[d] [KEYWORDTOKEN|SUGARTOKEN|SINGLELINECOMMENT] RIGHTBRACKET[d] ID COLON[d] STRING SEMICOLON[d]")]
+        "token : attribute* LEFTBRACKET[d] [SUGARTOKEN|SINGLELINECOMMENT] RIGHTBRACKET[d] ID COLON[d] STRING SEMICOLON[d]")]
     public ICLIModel OneArgToken(List<ICLIModel> attributes, Token<CLIToken> type, Token<CLIToken> id, Token<CLIToken> value, ParserContext context)
     {
         var tokenType = type.TokenID switch
@@ -161,6 +161,16 @@ public ICLIModel TwoArgToken(List<ICLIModel> attributes, Token<CLIToken> type, T
         return new TokenModel(attributes.Cast<AttributeModel>().ToList(), tokenType,id.Value,arg1.StringWithoutQuotes.Replace("\\\\","\\"), arg2.StringWithoutQuotes.Replace("\\\\","\\")) {Position = type.Position};
     }
 
+    [Production(
+        "token : attribute* LEFTBRACKET[d] [KEYWORDTOKEN] RIGHTBRACKET[d] ID COLON[d]  STRING* SEMICOLON[d]")]
+    public ICLIModel ManyArgToken(List<ICLIModel> attributes, Token<CLIToken> type, Token<CLIToken> id, List<Token<CLIToken>> args, ParserContext context)
+    {
+        var tokenType = GenericToken.KeyWord;
+        context.AddEnumName(id.Value);
+        var aargs = args.Select(arg => arg.StringWithoutQuotes.Replace("\\\\", "\\")).ToArray();
+        return new TokenModel(attributes.Cast<AttributeModel>().ToList(), tokenType,id.Value,aargs) {Position = type.Position};
+    }
+
     [Production(
         "token :attribute* LEFTBRACKET[d] DATETOKEN[d] RIGHTBRACKET[d] ID COLON[d] [DDMMYYYY|YYYYMMDD] CHAR SEMICOLON[d]")]
     public ICLIModel DateToken(List<ICLIModel> attributes, Token<CLIToken> id, Token<CLIToken> dateType, Token<CLIToken> separator, ParserContext context)