#121 : get rid of lexerexception

b3b00 · Jun 7, 2019 · 84dba0e · 84dba0e
1 parent 0f8c681
commit 84dba0e
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 64 deletions.
diff --git a/sly/lexer/GenericLexer.cs b/sly/lexer/GenericLexer.cs
@@ -93,24 +93,23 @@ public void AddCallBack(IN token, Func<Token<IN>, Token<IN>> callback)
             CallBacks[token] = callback;
         }
 
-        public void AddDefinition(TokenDefinition<IN> tokenDefinition)
-        {
-            throw new NotImplementedException();
-        }
+        public void AddDefinition(TokenDefinition<IN> tokenDefinition) {}
 
         public LexerResult<IN> Tokenize(string source)
         {
             LexerResult<IN> result = null;
             var tokens = new List<Token<IN>>();
             FSMMatch<GenericToken> r = null;
-            try
-            {
-                r = LexerFsm.Run(source, 0);
-            }
-            catch (LexerException lexErr)
+
+            r = LexerFsm.Run(source, 0);
+            if (!r.IsSuccess && !r.IsEOS)
             {
-                return new LexerResult<IN>(lexErr.Error);
+                var resultPosition = r.Result.Position;
+                LexicalError error =
+                    new LexicalError(resultPosition.Line, resultPosition.Column, r.Result.CharValue);  
+                return new LexerResult<IN>(error);
             }
+
             while (r.IsSuccess)
             {
                 var transcoded = Transcode(r);
@@ -121,14 +120,17 @@ public LexerResult<IN> Tokenize(string source)
                 }
 
                 tokens.Add(transcoded);
-                try {
+
                 r = LexerFsm.Run(source);
-                }
-                catch (LexerException lexErr)
+                if (!r.IsSuccess && !r.IsEOS)
                 {
-                    return new LexerResult<IN>(lexErr.Error);
+                    var resultPosition = r.Result.Position;
+                    LexicalError error =
+                        new LexicalError(resultPosition.Line, resultPosition.Column, r.Result.CharValue);  
+                    return new LexerResult<IN>(error);
                 }
-
+
+
                 if (r.IsSuccess && r.Result.IsComment) ConsumeComment(r.Result, source);
             }
 

diff --git a/sly/lexer/LexerException.cs b/sly/lexer/LexerException.cs
diff --git a/sly/lexer/fsm/FSMLexer.cs b/sly/lexer/fsm/FSMLexer.cs
@@ -262,8 +262,12 @@ public FSMMatch<N> Run( ReadOnlyMemory<char> source, int start)
                     else
                     {
                         if (!successes.Any() && CurrentPosition < source.Length)
-                            throw new LexerException(new LexicalError(CurrentLine, CurrentColumn,
-                                source.At(CurrentPosition)));
+                        {
+                            var errorChar = source.Slice(CurrentPosition, 1);
+                            var errorPosition = new TokenPosition(CurrentPosition, CurrentLine, CurrentColumn);
+                            var ko = new FSMMatch<N>(false, default(N), errorChar, errorPosition, -1); 
+                            return ko;
+                        }
                     }
                 }
             }

diff --git a/sly/lexer/fsm/FSMMatch.cs b/sly/lexer/fsm/FSMMatch.cs
@@ -11,27 +11,35 @@ public class FSMMatch<N>
         public Dictionary<string, object> Properties { get; set; }
 
         public bool IsSuccess { get; set; }
+
+        public bool IsEOS { get; set; }
 
         public Token<N> Result { get; set; }
 
         public int NodeId { get; set; }
         public FSMMatch(bool success)
         {
             IsSuccess = success;
+            if (!IsSuccess)
+            {
+                IsEOS = true;
+            }
         }
 
         public FSMMatch(bool success, N result, string value, TokenPosition position,int nodeId) : this(success,result,new ReadOnlyMemory<char>(value.ToCharArray()),position,nodeId )
         {
-
         }
 
         public FSMMatch(bool success, N result, ReadOnlyMemory<char> value, TokenPosition position, int nodeId)
         {
             Properties = new Dictionary<string, object>();
             IsSuccess = success;
             NodeId = nodeId;
+            IsEOS = false;
             Result = new Token<N>(result, value, position);
         }
+
+
 
 
     }

diff --git a/sly/parser/parser/Parser.cs b/sly/parser/parser/Parser.cs
@@ -56,52 +56,45 @@ public Parser(ISyntaxParser<IN, OUT> syntaxParser, SyntaxTreeVisitor<IN, OUT> vi
         {
             return ParseWithContext(source,new NoContext(),startingNonTerminal);
         }
-        
-        
+
+
         public ParseResult<IN, OUT> ParseWithContext(string source, object context, string startingNonTerminal = null)
         {
             ParseResult<IN, OUT> result = null;
-            try
-            {
-                var lexingResult = Lexer.Tokenize(source);
-                if (lexingResult.IsError)
-                {
-                    result = new ParseResult<IN, OUT>();
-                    result.IsError = true;
-                    result.Errors = new List<ParseError>();
-                    result.Errors.Add(lexingResult.Error);
-                    return result;
-                }
-                var tokens = lexingResult.Tokens;
-                var position = 0;
-                var tokensWithoutComments = new List<Token<IN>>();
-                for (var i = 0; i < tokens.Count; i++)
-                {
-                    var token = tokens[i];
-                    if (!token.IsComment)
-                    {
-                        token.PositionInTokenFlow = position;
-                        tokensWithoutComments.Add(token);
-                        position++;
-                    }
-                }
 
-                result = ParseWithContext(tokensWithoutComments, context, startingNonTerminal);
-            }
-            catch (LexerException e)
+            var lexingResult = Lexer.Tokenize(source);
+            if (lexingResult.IsError)
             {
                 result = new ParseResult<IN, OUT>();
                 result.IsError = true;
                 result.Errors = new List<ParseError>();
-                result.Errors.Add(e.Error);
+                result.Errors.Add(lexingResult.Error);
+                return result;
             }
 
+            var tokens = lexingResult.Tokens;
+            var position = 0;
+            var tokensWithoutComments = new List<Token<IN>>();
+            for (var i = 0; i < tokens.Count; i++)
+            {
+                var token = tokens[i];
+                if (!token.IsComment)
+                {
+                    token.PositionInTokenFlow = position;
+                    tokensWithoutComments.Add(token);
+                    position++;
+                }
+            }
+
+            result = ParseWithContext(tokensWithoutComments, context, startingNonTerminal);
+
+
             return result;
         }
 
 
-       
-        
+
+
         public ParseResult<IN, OUT> ParseWithContext(IList<Token<IN>> tokens, object parsingContext = null, string startingNonTerminal = null)
         {
             var result = new ParseResult<IN, OUT>();