Add token parser for bot interpreter

EOBot/EOBot.csproj

@@ -93,6 +93,10 @@
     <Compile Include="DependencyMaster.cs" />
     <Compile Include="IBot.cs" />
     <Compile Include="IBotFactory.cs" />
+    <Compile Include="Interpreter\BotInterpreter.cs" />
+    <Compile Include="Interpreter\BotToken.cs" />
+    <Compile Include="Interpreter\BotTokenParser.cs" />
+    <Compile Include="Interpreter\BotTokenType.cs" />
     <Compile Include="NamesList.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -122,6 +126,9 @@
     <None Include="app.config" />
     <None Include="packages.config" />
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Interpreter\States\" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.

EOBot/Interpreter/BotInterpreter.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace EOBot.Interpreter
+{
+    public class BotInterpreter
+    {
+        private readonly Dictionary<string, string> _symbolTable;
+        private readonly BotTokenParser _parser;
+
+        public BotInterpreter(string filePath)
+            : this(File.OpenText(filePath))
+        {
+        }
+
+        public BotInterpreter(StreamReader inputStream)
+        {
+            _parser = new BotTokenParser(inputStream);
+        }
+
+        public List<BotToken> Parse()
+        {
+            _parser.Reset();
+
+            var retList = new List<BotToken>();
+
+            BotToken nextToken;
+            do
+            {
+                nextToken = _parser.GetNextToken();
+                if (nextToken.TokenType == BotTokenType.Error)
+                {
+                    ConsoleHelper.WriteMessage(ConsoleHelper.Type.Error, $"Error at line {_parser.LineNumber} column {_parser.Column}: token value {nextToken.TokenValue}", ConsoleColor.Red);
+                    throw new InvalidOperationException("Unable to parse input");
+                }
+
+                retList.Add(nextToken);
+            } while (nextToken.TokenType != BotTokenType.EOF);
+
+            return retList;
+        }
+
+        public async Task Run(List<BotToken> tokens)
+        {
+        }
+    }
+}

EOBot/Interpreter/BotToken.cs

@@ -0,0 +1,14 @@
+namespace EOBot.Interpreter
+{
+    public class BotToken
+    {
+        public BotTokenType TokenType { get; }
+        public string TokenValue { get; }
+
+        public BotToken(BotTokenType tokenType, string tokenValue)
+        {
+            TokenType = tokenType;
+            TokenValue = tokenValue;
+        }
+    }
+}

EOBot/Interpreter/BotTokenParser.cs

@@ -0,0 +1,209 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace EOBot.Interpreter
+{
+    public sealed class BotTokenParser : IDisposable
+    {
+        private static readonly HashSet<string> Keywords = new HashSet<string>
+        {
+            "if",
+            "while",
+            "goto"
+        };
+
+        private readonly StreamReader _inputStream;
+        private readonly bool _streamNeedsDispose;
+
+        public int LineNumber { get; private set; }
+
+        public int Column { get; private set; }
+
+        public BotTokenParser(string filePath)
+            : this(File.OpenText(filePath))
+        {
+            _streamNeedsDispose = true;
+        }
+
+        public BotTokenParser(StreamReader inputStream)
+        {
+            _inputStream = inputStream;
+            LineNumber = 1;
+            Column = 1;
+        }
+
+        public void Reset()
+        {
+            _inputStream.BaseStream.Seek(0, SeekOrigin.Begin);
+            LineNumber = 1;
+            Column = 1;
+        }
+
+        public BotToken GetNextToken()
+        {
+            if (_inputStream.EndOfStream)
+                return new BotToken(BotTokenType.EOF, string.Empty);
+
+            char inputChar;
+            do
+            {
+                inputChar = Read();
+
+                if (inputChar == '\n')
+                {
+                    LineNumber++;
+                    Column = 1;
+                    return new BotToken(BotTokenType.NewLine, inputChar.ToString());
+                }
+
+                if (inputChar == '/' && !_inputStream.EndOfStream && (char)_inputStream.Peek() == '*')
+                {
+                    // skip the comment: block format
+                    do
+                    {
+                        inputChar = Read();
+                    } while (!(inputChar == '*' && !_inputStream.EndOfStream && _inputStream.Peek() == '/'));
+
+                    // skip the slash ending the comment and set input char to the character after the comment
+                    Read();
+                    inputChar = Read();
+                }
+                else if (inputChar == '/' && !_inputStream.EndOfStream && (char)_inputStream.Peek() == '/')
+                {
+                    // skip the comment: line format
+                    do
+                    {
+                        inputChar = Read();
+                    } while (inputChar != '\n');
+
+                    LineNumber++;
+                    Column = 1;
+                    return new BotToken(BotTokenType.NewLine, inputChar.ToString());
+                }
+            } while (!_inputStream.EndOfStream && char.IsWhiteSpace(inputChar));
+
+            if (_inputStream.EndOfStream)
+                return new BotToken(BotTokenType.EOF, string.Empty);
+
+            if (char.IsLetter(inputChar))
+            {
+                var identifier = inputChar.ToString();
+                while (char.IsLetterOrDigit((char)_inputStream.Peek()))
+                    identifier += Read();
+
+                var type = Keywords.Contains(identifier)
+                        ? BotTokenType.Keyword
+                        : BotTokenType.Label;
+
+                if (type == BotTokenType.Label && (char)_inputStream.Peek() == ':')
+                {
+                    Read();
+                }
+
+                return new BotToken(type, identifier);
+            }
+            else if (char.IsDigit(inputChar))
+            {
+                var number = inputChar.ToString();
+                while (char.IsDigit((char)_inputStream.Peek()))
+                    number += Read();
+                return new BotToken(BotTokenType.Literal, number);
+            }
+            else
+            {
+                switch(inputChar)
+                {
+                    case '(': return new BotToken(BotTokenType.LParen, inputChar.ToString());
+                    case ')': return new BotToken(BotTokenType.RParen, inputChar.ToString());
+                    case '{': return new BotToken(BotTokenType.LBrace, inputChar.ToString());
+                    case '}': return new BotToken(BotTokenType.RBrace, inputChar.ToString());
+                    case '[': return new BotToken(BotTokenType.LBracket, inputChar.ToString());
+                    case ']': return new BotToken(BotTokenType.RBracket, inputChar.ToString());
+                    case '"':
+                        {
+                            var stringLiteral = string.Empty;
+                            while ((char)_inputStream.Peek() != '"')
+                                stringLiteral += Read();
+                            Read();
+                            return new BotToken(BotTokenType.Literal, stringLiteral);
+                        }
+                    case '=':
+                        {
+                            switch((char)_inputStream.Peek())
+                            {
+                                case '=':
+                                    var nextChar = Read();
+                                    return new BotToken(BotTokenType.EqualOperator, inputChar.ToString() + nextChar);
+                                default:
+                                    return new BotToken(BotTokenType.AssignOperator, inputChar.ToString());
+                            }
+                        }
+                    case '!':
+                        {
+                            var nextChar = Read();
+                            if (nextChar != '=')
+                                return new BotToken(BotTokenType.Error, inputChar.ToString() + nextChar);
+                            return new BotToken(BotTokenType.NotEqualOperator, inputChar.ToString() + nextChar);
+                        }
+                    case '>':
+                        {
+                            switch ((char)_inputStream.Peek())
+                            {
+                                case '=':
+                                    var nextChar = Read();
+                                    return new BotToken(BotTokenType.GreaterThanEqOperator, inputChar.ToString() + nextChar);
+                                default:
+                                    return new BotToken(BotTokenType.GreaterThanOperator, inputChar.ToString());
+                            }
+                        }
+                    case '<':
+                        {
+                            switch ((char)_inputStream.Peek())
+                            {
+                                case '=':
+                                    var nextChar = Read();
+                                    return new BotToken(BotTokenType.LessThanEqOperator, inputChar.ToString() + nextChar);
+                                default:
+                                    return new BotToken(BotTokenType.LessThanOperator, inputChar.ToString());
+                            }
+                        }
+                    case '$':
+                        {
+                            if (_inputStream.EndOfStream)
+                                return new BotToken(BotTokenType.Error, inputChar.ToString());
+
+                            // ensure identifier starts with letter or underscore before getting identifier name
+                            inputChar = (char)_inputStream.Peek();
+                            if (!char.IsLetter(inputChar) && inputChar != '_')
+                                return new BotToken(BotTokenType.Error, inputChar.ToString());
+
+                            var identifier = string.Empty;
+                            do
+                            {
+                                inputChar = (char)Read();
+                                identifier += inputChar;
+                            } while (!_inputStream.EndOfStream && (char.IsLetterOrDigit(inputChar) || inputChar == '_'));
+
+                            return new BotToken(BotTokenType.Identifier, identifier);
+                        }
+                    default: return new BotToken(BotTokenType.Error, inputChar.ToString());
+                }
+            }
+        }
+
+        public void Dispose()
+        {
+            if (_streamNeedsDispose)
+                _inputStream.Dispose();
+
+            GC.SuppressFinalize(this);
+        }
+
+        private char Read()
+        {
+            Column++;
+            return (char)_inputStream.Read();
+        }
+    }
+}

EOBot/Interpreter/BotTokenType.cs

@@ -0,0 +1,26 @@
+namespace EOBot.Interpreter
+{
+    public enum BotTokenType
+    {
+        EOF,
+        LParen,
+        RParen,
+        LBrace,
+        RBrace,
+        LBracket,
+        RBracket,
+        Keyword,
+        Identifier,
+        Label,
+        Literal,
+        AssignOperator,
+        EqualOperator,
+        LessThanOperator,
+        LessThanEqOperator,
+        GreaterThanOperator,
+        GreaterThanEqOperator,
+        NotEqualOperator,
+        NewLine,
+        Error
+    }
+}