#486 - Tokenize Keywords separately

sys27 · Nov 21, 2021 · 69d68ef · 69d68ef
1 parent ecc78a9
commit 69d68ef
Show file tree

Hide file tree

Showing 3 changed files with 91 additions and 66 deletions.
diff --git a/xFunc.Maths/Tokenization/Lexer.IdToken.cs b/xFunc.Maths/Tokenization/Lexer.IdToken.cs
@@ -1,9 +1,6 @@
 // Copyright (c) Dmytro Kyshchenko. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using System.Runtime.CompilerServices;
-using static xFunc.Maths.Tokenization.TokenKind;
-
 namespace xFunc.Maths.Tokenization;
 
 /// <summary>
@@ -21,71 +18,9 @@ private bool CreateIdToken()
             endIndex++;
 
         var id = function[..endIndex];
-
-        var kind = Empty;
-
-        if (Compare(id, "true"))
-            kind = TrueKeyword;
-        else if (Compare(id, "false"))
-            kind = FalseKeyword;
-        else if (Compare(id, "degrees") || Compare(id, "degree") || Compare(id, "deg"))
-            kind = DegreeKeyword;
-        else if (Compare(id, "radians") || Compare(id, "radian") || Compare(id, "rad"))
-            kind = RadianKeyword;
-        else if (Compare(id, "gradians") || Compare(id, "gradian") || Compare(id, "grad"))
-            kind = GradianKeyword;
-        else if (Compare(id, "def") || Compare(id, "define"))
-            kind = DefineKeyword;
-        else if (Compare(id, "undef") || Compare(id, "undefine"))
-            kind = UndefineKeyword;
-        else if (Compare(id, "if"))
-            kind = IfKeyword;
-        else if (Compare(id, "for"))
-            kind = ForKeyword;
-        else if (Compare(id, "while"))
-            kind = WhileKeyword;
-        else if (Compare(id, "nand"))
-            kind = NAndKeyword;
-        else if (Compare(id, "nor"))
-            kind = NOrKeyword;
-        else if (Compare(id, "and"))
-            kind = AndKeyword;
-        else if (Compare(id, "or"))
-            kind = OrKeyword;
-        else if (Compare(id, "xor"))
-            kind = XOrKeyword;
-        else if (Compare(id, "not"))
-            kind = NotKeyword;
-        else if (Compare(id, "eq"))
-            kind = EqKeyword;
-        else if (Compare(id, "impl"))
-            kind = ImplKeyword;
-        else if (Compare(id, "mod"))
-            kind = ModKeyword;
-        else if (Compare(id, "W"))
-            kind = WattKeyword;
-        else if (Compare(id, "kW"))
-            kind = KilowattKeyword;
-        else if (Compare(id, "hp"))
-            kind = HorsepowerKeyword;
-        else if (Compare(id, "°C"))
-            kind = CelsiusKeyword;
-        else if (Compare(id, "°F"))
-            kind = FahrenheitKeyword;
-        else if (Compare(id, "K"))
-            kind = KelvinKeyword;
-
         function = function[endIndex..];
-
-        if (kind == Empty)
-            Current = Token.Id(id.ToString().ToLowerInvariant()); // TODO:
-        else
-            Current = Token.Create(kind);
+        Current = Token.Id(id.ToString().ToLowerInvariant()); // TODO:
 
         return true;
     }
-
-    [MethodImpl(MethodImplOptions.AggressiveInlining)]
-    private static bool Compare(ReadOnlySpan<char> id, string str)
-        => id.Equals(str, StringComparison.OrdinalIgnoreCase);
 }
diff --git a/xFunc.Maths/Tokenization/Lexer.KeywordToken.cs b/xFunc.Maths/Tokenization/Lexer.KeywordToken.cs
@@ -0,0 +1,89 @@
+// Copyright (c) Dmytro Kyshchenko. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Runtime.CompilerServices;
+using static xFunc.Maths.Tokenization.TokenKind;
+
+namespace xFunc.Maths.Tokenization;
+
+/// <summary>
+/// The lexer for mathematical expressions.
+/// </summary>
+internal ref partial struct Lexer
+{
+    private bool CreateKeywordToken()
+    {
+        if (!char.IsLetter(function[0]) && function[0] != '°')
+            return false;
+
+        var endIndex = 1;
+        while (endIndex < function.Length && char.IsLetterOrDigit(function[endIndex]))
+            endIndex++;
+
+        var keyword = function[..endIndex];
+        var kind = Empty;
+
+        if (Compare(keyword, "true"))
+            kind = TrueKeyword;
+        else if (Compare(keyword, "false"))
+            kind = FalseKeyword;
+        else if (Compare(keyword, "degrees") || Compare(keyword, "degree") || Compare(keyword, "deg"))
+            kind = DegreeKeyword;
+        else if (Compare(keyword, "radians") || Compare(keyword, "radian") || Compare(keyword, "rad"))
+            kind = RadianKeyword;
+        else if (Compare(keyword, "gradians") || Compare(keyword, "gradian") || Compare(keyword, "grad"))
+            kind = GradianKeyword;
+        else if (Compare(keyword, "def") || Compare(keyword, "define"))
+            kind = DefineKeyword;
+        else if (Compare(keyword, "undef") || Compare(keyword, "undefine"))
+            kind = UndefineKeyword;
+        else if (Compare(keyword, "if"))
+            kind = IfKeyword;
+        else if (Compare(keyword, "for"))
+            kind = ForKeyword;
+        else if (Compare(keyword, "while"))
+            kind = WhileKeyword;
+        else if (Compare(keyword, "nand"))
+            kind = NAndKeyword;
+        else if (Compare(keyword, "nor"))
+            kind = NOrKeyword;
+        else if (Compare(keyword, "and"))
+            kind = AndKeyword;
+        else if (Compare(keyword, "or"))
+            kind = OrKeyword;
+        else if (Compare(keyword, "xor"))
+            kind = XOrKeyword;
+        else if (Compare(keyword, "not"))
+            kind = NotKeyword;
+        else if (Compare(keyword, "eq"))
+            kind = EqKeyword;
+        else if (Compare(keyword, "impl"))
+            kind = ImplKeyword;
+        else if (Compare(keyword, "mod"))
+            kind = ModKeyword;
+        else if (Compare(keyword, "W"))
+            kind = WattKeyword;
+        else if (Compare(keyword, "kW"))
+            kind = KilowattKeyword;
+        else if (Compare(keyword, "hp"))
+            kind = HorsepowerKeyword;
+        else if (Compare(keyword, "°C"))
+            kind = CelsiusKeyword;
+        else if (Compare(keyword, "°F"))
+            kind = FahrenheitKeyword;
+        else if (Compare(keyword, "K"))
+            kind = KelvinKeyword;
+
+        if (kind == Empty)
+            return false;
+
+        function = function[endIndex..];
+        Current = Token.Create(kind);
+
+        return true;
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static bool Compare(ReadOnlySpan<char> id, string str)
+        => id.Equals(str, StringComparison.OrdinalIgnoreCase);
+}
diff --git a/xFunc.Maths/Tokenization/Lexer.cs b/xFunc.Maths/Tokenization/Lexer.cs
@@ -38,6 +38,7 @@ public bool MoveNext()
         while (HasSymbols())
         {
             var result = CreateNumberToken() ||
+                         CreateKeywordToken() ||
                          CreateIdToken() ||
                          CreateOperatorToken() ||
                          CreateSymbol() ||