bugfix #175

b3b00 · May 15, 2020 · d49827b · d49827b
1 parent 4261aa9
commit d49827b
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 5 deletions.
diff --git a/ParserTests/lexer/GenericLexerTests.cs b/ParserTests/lexer/GenericLexerTests.cs
@@ -664,10 +664,12 @@ public void TestManyString()
             var tok1 = r.Tokens[0];
             Assert.Equal(ManyString.STRING, tok1.TokenID);
             Assert.Equal(expectString1, tok1.Value);
+            Assert.Equal('"',tok1.StringDelimiter);
 
             var tok2 = r.Tokens[1];
             Assert.Equal(ManyString.STRING, tok2.TokenID);
             Assert.Equal(expectString2, tok2.Value);
+            Assert.Equal('\'',tok2.StringDelimiter);
         }
 
         [Fact]

diff --git a/samples/ParserExample/Program.cs b/samples/ParserExample/Program.cs
@@ -21,9 +21,18 @@
 using sly.parser.syntax.grammar;
 using sly.buildresult;
 using sly.parser.generator.visitor;
+using Xunit;
 
 namespace ParserExample
 {
+
+    public enum ManyString
+    {
+        [Lexeme(GenericToken.String, "'", "'")]
+        [Lexeme(GenericToken.String)]
+        STRING
+    }
+
     public enum TokenType
     {
         [Lexeme("a")] a = 1,
@@ -530,21 +539,46 @@ public static void TestAssociativityFactorExpressionParser()
             Console.WriteLine($"{r.IsOk} : {r.Result}");
             ;
         }
+
+        public static void TestManyString()
+        {
+            var lexerRes = LexerBuilder.BuildLexer(new BuildResult<ILexer<ManyString>>());
+            Assert.False(lexerRes.IsError);
+            var lexer = lexerRes.Result;
+            var string1 = "\"hello \\\"world \"";
+            var expectString1 = "\"hello \"world \"";
+            var string2 = "'that''s it'";
+            var expectString2 = "'that's it'";
+            var source1 = $"{string1} {string2}";
+            var r = lexer.Tokenize(source1);
+            Assert.True(r.IsOk);
+            Assert.Equal(3, r.Tokens.Count);
+            var tok1 = r.Tokens[0];
+            Assert.Equal(ManyString.STRING, tok1.TokenID);
+            Assert.Equal(expectString1, tok1.Value);
+            Assert.Equal('"',tok1.StringDelimiter);
+
+            var tok2 = r.Tokens[1];
+            Assert.Equal(ManyString.STRING, tok2.TokenID);
+            Assert.Equal(expectString2, tok2.Value);
+            Assert.Equal('\'',tok2.StringDelimiter);
+        }
 
         private static void Main(string[] args)
         {
             //TestContextualParser();
             //TestTokenCallBacks();
             //test104();
-            //testJSON();
+            // testJSON();
            //TestGrammarParser();
             // TestGraphViz();
 
             // TestGraphViz();
             // TestChars();
             //TestAssociativityFactorExpressionParser();
-            TestFactorial();
-            TestThreadsafeGeneric();
+            // TestFactorial();
+            //TestThreadsafeGeneric();
+            TestManyString();
 
         }
     }

diff --git a/sly/lexer/GenericLexer.cs b/sly/lexer/GenericLexer.cs
@@ -574,7 +574,8 @@ public void AddStringLexem(IN token, string stringDelimiter, string escapeDelimi
 
                 match.Result.SpanValue = value;
 
-
+                match.StringDelimiterChar = stringDelimiterChar;
+                match.IsString = true;
                 if (stringDelimiterChar != escapeStringDelimiterChar)
                 {
                     match.Result.SpanValue = diffCharEscaper(escapeStringDelimiterChar,stringDelimiterChar, match.Result.SpanValue);
@@ -756,7 +757,7 @@ public Token<IN> Transcode(FSMMatch<GenericToken> match)
             tok.CommentType = inTok.CommentType;
             tok.Position = inTok.Position;
             tok.Discarded = inTok.Discarded;
-            tok.StringDelimiter = StringDelimiterChar;
+            tok.StringDelimiter = match.StringDelimiterChar;
             tok.TokenID = (IN) match.Properties[DerivedToken];
             tok.IsEOS = tok.TokenID.Equals(default(IN));
             return tok;

diff --git a/sly/lexer/fsm/FSMMatch.cs b/sly/lexer/fsm/FSMMatch.cs
@@ -7,6 +7,10 @@ public class FSMMatch<N>
     {
         public Dictionary<string, object> Properties { get; }
 
+        public bool IsString { get; set; }
+
+        public char StringDelimiterChar { get; set; }
+
         public bool IsSuccess { get; }
 
         public bool IsEOS { get; }