Add indentation token

BenWu · Sep 26, 2018 · 7e26d4a · 7e26d4a
1 parent f011d80
commit 7e26d4a
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 20 deletions.
diff --git a/README.md b/README.md
@@ -81,24 +81,13 @@ n 1
 
 ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ n ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ m
 
-### logical operators follow a comparator
-
-#### AND
-
-¯\\\_(ツ)\_/¯
-
-#### OR
-
-¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯
-
-
 ### conditional statements start with ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯
 
 ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ comparator 
 	statement1
 ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ comparator2
 	statement2
-¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ <- optional default statement
+¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ ¯\\\_(ツ)\_/¯ <- optional default statement
 	statement2
 ¯\\\_(ツ)\_/¯ <- end conditional
 

diff --git a/shrug_lang/shrug_token.py b/shrug_lang/shrug_token.py
@@ -6,11 +6,12 @@
 class TokenType(Enum):
     SHRUG = 1
     NUMBER = 2
-    BOOL = 7
     STRING = 3
     ID = 4
     EOL = 5
     INVALID = 6
+    BOOL = 7
+    INDENT = 8
 
 
 @dataclass(init=False)
@@ -22,7 +23,8 @@ def __init__(self, _type: TokenType, value: Union[int, str, None]=None):
         if not isinstance(_type, TokenType):
             raise TypeError('Token type must be of TokenType')
         must_have_value = {TokenType.NUMBER, TokenType.STRING, TokenType.ID,
-                           TokenType.INVALID, TokenType.BOOL}
+                           TokenType.INVALID, TokenType.BOOL,
+                           TokenType.INDENT}
         if _type in must_have_value and value is None:
             raise ValueError(f'Token type {_type} must have a value')
         self.type = _type

diff --git a/shrug_lang/tokenizer.py b/shrug_lang/tokenizer.py
@@ -6,8 +6,19 @@
 class Tokenizer:
     @staticmethod
     def parse_line(line: str) -> List[Token]:
-        unparsed_tokens = filter(None, Tokenizer.join_strings(line.split(' ')))
-        tokens = [Tokenizer.parse_token(unparsed) for unparsed in unparsed_tokens]
+        unparsed_tokens = Tokenizer.join_strings(line.split(' '))
+        indent_size = 0
+        if len(unparsed_tokens) > 1:
+            for c in unparsed_tokens[1:]:
+                if c == '':
+                    indent_size += 1
+        unparsed_tokens = filter(None, unparsed_tokens)
+        if indent_size > 0:
+            tokens = [Token(TokenType.INDENT, indent_size)]
+        else:
+            tokens = []
+        tokens += [Tokenizer.parse_token(unparsed)
+                   for unparsed in unparsed_tokens]
         tokens.append(Token(TokenType.EOL))
         return tokens
 

diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -1,19 +1,19 @@
 import unittest
 
-from shrug_lang.tokenizer import Tokenizer
 from shrug_lang.shrug_token import TokenType, Token
+from shrug_lang.tokenizer import Tokenizer
 
 
 class TestTokenizer(unittest.TestCase):
     """Line parser class should return correctly read tokens"""
 
     def test_empty_line(self):
         line = ''
-        empty_line = [Token(TokenType.EOL)]
-        self.assertEqual(empty_line, Tokenizer.parse_line(line))
+        self.assertEqual([Token(TokenType.EOL)], Tokenizer.parse_line(line))
 
         line = '     '
-        self.assertEqual(empty_line, Tokenizer.parse_line(line))
+        self.assertEqual([Token(TokenType.INDENT, 5), Token(TokenType.EOL)],
+                         Tokenizer.parse_line(line))
 
     def test_shrug(self):
         line = '¯\_(ツ)_/¯'