Small tokenizer refactoring

discoveryjs · Jul 25, 2019 · 17dd0bc · 17dd0bc
1 parent d15bea9
commit 17dd0bc
Showing 1 changed file with 4 additions and 7 deletions.
diff --git a/utils/tokenize.js b/utils/tokenize.js
@@ -184,24 +184,21 @@ module.exports = (input) => {
     const tokens = [];
 
     while (index < input.length) {
-        const matched = (
+        const token = (
             parseWhitespace(input, index) ||
             parseDelim(input, index) ||
             parseKeyword(input, index) ||
             parseString(input, index) ||
             parseNumber(input, index)
         );
 
-        if (!matched) {
+        if (!token) {
             break;
         }
 
-        tokens.push({
-            type: matched.type,
-            value: matched.value
-        });
+        tokens.push(token);
 
-        index += matched.value.length;
+        index += token.value.length;
     }
 
     return tokens;