Token content now consumed from actual input

ParrotFx · Jul 23, 2013 · 0f3b887 · 0f3b887
1 parent 7eb4bf9
commit 0f3b887
Show file tree

Hide file tree

Showing 12 changed files with 6 additions and 61 deletions.
diff --git a/src/Parrot/Lexer/AtToken.cs b/src/Parrot/Lexer/AtToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     internal class AtToken : Token
     {
-        public AtToken()
-        {
-            Content = "@";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.At; }

diff --git a/src/Parrot/Lexer/CloseBracesToken.cs b/src/Parrot/Lexer/CloseBracesToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     public class CloseBracesToken : Token
     {
-        public CloseBracesToken()
-        {
-            Content = "}";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.CloseBrace; }

diff --git a/src/Parrot/Lexer/CloseBracketToken.cs b/src/Parrot/Lexer/CloseBracketToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     public class CloseBracketToken : Token
     {
-        public CloseBracketToken()
-        {
-            Content = "]";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.CloseBracket; }

diff --git a/src/Parrot/Lexer/CloseParenthesisToken.cs b/src/Parrot/Lexer/CloseParenthesisToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     public class CloseParenthesisToken : Token
     {
-        public CloseParenthesisToken()
-        {
-            Content = ")";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.CloseParenthesis; }

diff --git a/src/Parrot/Lexer/CommaToken.cs b/src/Parrot/Lexer/CommaToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     internal class CommaToken : Token
     {
-        public CommaToken()
-        {
-            Content = ",";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.Comma; }

diff --git a/src/Parrot/Lexer/EqualToken.cs b/src/Parrot/Lexer/EqualToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     public class EqualToken : Token
     {
-        public EqualToken()
-        {
-            Content = "=";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.Equal; }

diff --git a/src/Parrot/Lexer/GreaterThanToken.cs b/src/Parrot/Lexer/GreaterThanToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     internal class GreaterThanToken : Token
     {
-        public GreaterThanToken()
-        {
-            Content = ">";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.GreaterThan; }

diff --git a/src/Parrot/Lexer/OpenBracesToken.cs b/src/Parrot/Lexer/OpenBracesToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     internal class OpenBracesToken : Token
     {
-        public OpenBracesToken()
-        {
-            Content = "{";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.OpenBrace; }

diff --git a/src/Parrot/Lexer/OpenBracketToken.cs b/src/Parrot/Lexer/OpenBracketToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     internal class OpenBracketToken : Token
     {
-        public OpenBracketToken()
-        {
-            Content = "[";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.OpenBracket; }

diff --git a/src/Parrot/Lexer/OpenParenthesisToken.cs b/src/Parrot/Lexer/OpenParenthesisToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     internal class OpenParenthesisToken : Token
     {
-        public OpenParenthesisToken()
-        {
-            Content = "(";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.OpenParenthesis; }

diff --git a/src/Parrot/Lexer/PlusToken.cs b/src/Parrot/Lexer/PlusToken.cs
@@ -2,11 +2,6 @@ namespace Parrot.Lexer
 {
     internal class PlusToken : Token
     {
-        public PlusToken()
-        {
-            Content = "+";
-        }
-
         public override TokenType Type
         {
             get { return TokenType.Plus; }

diff --git a/src/Parrot/Lexer/Tokenizer.cs b/src/Parrot/Lexer/Tokenizer.cs
@@ -42,17 +42,17 @@ public Tokenizer(Stream source)
             _reader = new StreamReader(source);
         }
 
-        private int Consume()
+        private char Consume()
         {
             _currentIndex += 1;
-            var character = _reader.Read();
+            int character = _reader.Read();
 
             if (character == -1)
             {
                 throw new EndOfStreamException();
             }
 
-            return character;
+            return (char) character;
         }
 
         private Token GetNextToken()
@@ -108,9 +108,9 @@ private Token initToken(Token token, Func<string> contentFunc)
 
         private Token ConsumeSingleCharToken(char currentCharacter)
         {
-            Consume();
             Token token = TokenFactory.Create(currentCharacter);
             token.Index = _currentIndex;
+            token.Content = Consume().ToString();
             return token;
         }
 
@@ -185,7 +185,7 @@ private string ConsumeWhitespace()
         private string ConsumeQuotedStringLiteral(char quote)
         {
             var sb = new StringBuilder();
-            sb.Append((char)Consume());
+            sb.Append(Consume());
             char currentCharacter = PeekCurrentCharacter();
 
             //extra quote for continuations
@@ -198,7 +198,7 @@ private string ConsumeQuotedStringLiteral(char quote)
                     sb.Append(currentCharacter);
                     currentCharacter = PeekCurrentCharacter();
                 }
-                sb.Append((char)Consume());
+                sb.Append(Consume());
                 if (_reader.Peek() != quote)
                 {
                     break;