bugfix #175 : line counter is using EOL tokens

ParserTests/lexer/GenericLexerTests.cs

@@ -275,6 +275,34 @@ public enum Issue138
         C,
     }
 
+    [Lexer(IgnoreEOL=false)]
+    public enum Issue177Generic
+    {
+        [Lexeme(GenericToken.SugarToken,"\r\n",IsLineEnding = true)]
+        EOL = 1,
+
+        [Lexeme(GenericToken.Int)]
+        INT = 2,
+
+        EOS = 0
+
+    }
+
+    [Lexer(IgnoreEOL=false)]
+    public enum Issue177Regex
+    {
+        [Lexeme("\r\n",IsLineEnding = true)]
+        EOL = 1,
+
+        [Lexeme("\\d+")]
+        INT = 2,
+
+        EOS = 0
+
+    }
+
+
+
     public class GenericLexerTests
     {
         [Fact]
@@ -850,6 +878,32 @@ public void TestIssue138()
             Assert.Equal("BB0", ToTokens(result));
         }
 
+        [Fact]
+        public void TestIssue177()
+        {
+            var res = LexerBuilder.BuildLexer(new BuildResult<ILexer<Issue177Generic>>());
+            Assert.False(res.IsError);
+            var lexer = res.Result;
+
+            var result = lexer.Tokenize(@"1
+2
+3");
+            Assert.True(result.IsOk);
+            Assert.Equal(6,result.Tokens.Count);
+            ;
+
+            var res2 = LexerBuilder.BuildLexer(new BuildResult<ILexer<Issue177Regex>>());
+            Assert.False(res.IsError);
+            var lexer2 = res.Result;
+
+            var result2 = lexer2.Tokenize(@"1
+2
+3");
+            Assert.True(result2.IsOk);
+            Assert.Equal(6,result2.Tokens.Count);
+            ;
+        }
+
         private static string ToTokens<T>(LexerResult<T> result) where T : struct
         {
             return result.Tokens.Aggregate(new StringBuilder(), (buf, token) => buf.Append(token.TokenID)).ToString();

sly/lexer/GenericLexer.cs

@@ -176,7 +176,6 @@ public LexerResult<IN> Tokenize(ReadOnlyMemory<char> memorySource)
                 r = LexerFsm.Run(memorySource,position);
                 if (!r.IsSuccess && !r.IsEOS)
                 {
-
                     var result = r.Result;
                     var error = new LexicalError(result.Position.Line, result.Position.Column, result.CharValue);
                     return new LexerResult<IN>(error);
@@ -187,6 +186,12 @@ public LexerResult<IN> Tokenize(ReadOnlyMemory<char> memorySource)
                     position = r.NewPosition;
                     position = ConsumeComment(r.Result, memorySource, position);
                 }
+
+                if (r.IsLineEnding)
+                {
+                    position = position.Clone();
+                    position.Line++;
+                }
             }
 
             var eos = new Token<IN>();
@@ -679,7 +684,7 @@ public void AddCharLexem(IN token, string charDelimiter, string escapeDelimiterC
 
         }
 
-        public void AddSugarLexem(IN token, string specialValue)
+        public void AddSugarLexem(IN token, string specialValue, bool isLineEnding = false)
         {
             if (char.IsLetter(specialValue[0]))
                 throw new InvalidLexerException(
@@ -692,7 +697,7 @@ public void AddSugarLexem(IN token, string specialValue)
 
             FSMBuilder.GoTo(start);
             for (var i = 0; i < specialValue.Length; i++) FSMBuilder.SafeTransition(specialValue[i]);
-            FSMBuilder.End(GenericToken.SugarToken)
+            FSMBuilder.End(GenericToken.SugarToken, isLineEnding)
                 .CallBack(callback);
         }
 
@@ -750,6 +755,7 @@ public Token<IN> Transcode(FSMMatch<GenericToken> match)
             tok.Discarded = inTok.Discarded;
             tok.StringDelimiter = StringDelimiterChar;
             tok.TokenID = (IN) match.Properties[DerivedToken];
+            tok.IsLineEnding = match.IsLineEnding;
             tok.IsEOS = tok.TokenID.Equals(default(IN));
             return tok;
         }

sly/lexer/LexerBuilder.cs

@@ -232,7 +232,7 @@ private static IEnumerable<char[]> ParseIdentifierPattern(string pattern)
                         {
                             foreach (var param in lexeme.GenericTokenParameters)
                             {
-                                lexer.AddSugarLexem(tokenID, param);
+                                lexer.AddSugarLexem(tokenID, param, lexeme.IsLineEnding);
                             }
                         }
 

sly/lexer/Token.cs

@@ -134,6 +134,7 @@ public  double DoubleValue
 
         public bool End { get; set; }
         public static T DefTok { get; set; }
+        public bool IsLineEnding { get; set; }
 
         public static Token<T> Empty()
         {

sly/lexer/fsm/FSMLexer.cs

@@ -168,7 +168,7 @@ public FSMMatch<N> Run(ReadOnlyMemory<char> source, LexerPosition lexerPosition)
                 if (currentNode.IsEnd)
                 {
                     // Remember the possible match
-                    result = new FSMMatch<N>(true, currentNode.Value, currentValue, position, currentNode.Id,lexerPosition);
+                    result = new FSMMatch<N>(true, currentNode.Value, currentValue, position, currentNode.Id,lexerPosition, currentNode.IsLineEnding);
                 }
 
                 lexerPosition.Index++;
@@ -198,7 +198,7 @@ public FSMMatch<N> Run(ReadOnlyMemory<char> source, LexerPosition lexerPosition)
             }
 
             var errorChar = source.Slice(lexerPosition.Index, 1);
-            var ko = new FSMMatch<N>(false, default(N), errorChar, lexerPosition, -1,lexerPosition);
+            var ko = new FSMMatch<N>(false, default(N), errorChar, lexerPosition, -1,lexerPosition, false);
             return ko;
         }
 

sly/lexer/fsm/FSMLexerBuilder.cs

@@ -107,14 +107,15 @@ public FSMLexerBuilder<N> WhiteSpace(char[] spaceChars)
 
         #region NODES
 
-        public FSMLexerBuilder<N> End(N nodeValue)
+        public FSMLexerBuilder<N> End(N nodeValue, bool isLineEnding = false)
         {
             if (Fsm.HasState(CurrentState))
             {
                 var node = Fsm.GetNode(CurrentState);
 
                 node.IsEnd = true;
                 node.Value = nodeValue;
+                node.IsLineEnding = isLineEnding;
             }
 
             return this;

sly/lexer/fsm/FSMMatch.cs

@@ -16,25 +16,33 @@ public class FSMMatch<N>
         public int NodeId { get; }
 
         public LexerPosition NewPosition { get; }
+
+        public bool IsLineEnding { get; set; }
 
         public FSMMatch(bool success)
         {
             IsSuccess = success;
             IsEOS = !success;
         }
+
+
 
-        public FSMMatch(bool success, N result, string value, LexerPosition position, int nodeId, LexerPosition newPosition)
-            : this(success, result, new ReadOnlyMemory<char>(value.ToCharArray()), position, nodeId,newPosition)
+        public FSMMatch(bool success, N result, string value, LexerPosition position, int nodeId, LexerPosition newPosition, bool isLineEnding)
+            : this(success, result, new ReadOnlyMemory<char>(value.ToCharArray()), position, nodeId,newPosition,isLineEnding)
         { }
 
-        public FSMMatch(bool success, N result, ReadOnlyMemory<char> value, LexerPosition position, int nodeId, LexerPosition newPosition)
+        public FSMMatch(bool success, N result, ReadOnlyMemory<char> value, LexerPosition position, int nodeId,
+            LexerPosition newPosition, bool isLineEnding)
         {
             Properties = new Dictionary<string, object>();
             IsSuccess = success;
             NodeId = nodeId;
             IsEOS = false;
             Result = new Token<N>(result, value, position);
             NewPosition = newPosition;
+            IsLineEnding = isLineEnding;
         }
+
+
     }
 }

sly/lexer/fsm/FSMNode.cs

@@ -15,5 +15,6 @@ internal FSMNode(N value)
 
         internal bool IsStart { get; set; } = false;
         public string Mark { get; internal set; }
+        public bool IsLineEnding { get; set; }
     }
 }