Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Port Morpher to Magpie.

  • Loading branch information...
commit f38d8ba352c295ba490d97c935562916d2bf02ce 1 parent 75aa2a9
@munificent authored
Showing with 56 additions and 3 deletions.
  1. +3 −1 example/LexerTest.mag
  2. +53 −2 lib/magpie/syntax/lexer.mag
View
4 example/LexerTest.mag
@@ -9,9 +9,11 @@ import io.path
def lexFile(path)
open(path) use with
val lexer = Lexer new(it read)
+ val morpher = Morpher new(tokens: lexer)
+
var tokens = 0
var errors = 0
- for token in lexer do
+ for token in morpher do
tokens = tokens + 1
match token type
case == TokenType.ERROR then
View
55 lib/magpie/syntax/lexer.mag
@@ -1,6 +1,6 @@
import magpie.syntax
-// TODO(bob): Move to somewhere more appropriate:
+// TODO(bob): Move to somewhere more appropriate then here
def (this is TokenType) toString
this name
@@ -383,7 +383,7 @@ def (c is String) _isOperator
end
defclass LexerIterator
- val lexer is Lexer
+ val lexer
var current = nothing
end
@@ -399,3 +399,54 @@ end
def (this is Lexer) iterate
LexerIterator new(lexer: this)
end
+
+// TODO(bob): Unify with Lexer and Annotator.
+defclass Morpher
+ val tokens
+ var eatLines is Bool = true
+end
+
+def (this is Morpher) readToken
+ while true do
+ val token = this tokens readToken
+ var done = true
+
+ match token type
+ // Ignore non-semantic tokens.
+ case == TokenType.WHITESPACE then done = false
+ case == TokenType.BLOCK_COMMENT then done = false
+ case == TokenType.LINE_COMMENT then done = false
+
+ // Ignore lines after tokens that can't end an expression.
+ case == TokenType.BACKTICK then this eatLines = true
+ case == TokenType.COMMA then this eatLines = true
+ case == TokenType.FIELD then this eatLines = true
+ case == TokenType.LEFT_PAREN then this eatLines = true
+ case == TokenType.LEFT_BRACKET then this eatLines = true
+ case == TokenType.LEFT_BRACE then this eatLines = true
+
+ case == TokenType.LINE_CONTINUATION then
+ this eatLines = true
+ done = false
+
+ case == TokenType.LINE then
+ if this eatLines then
+ done = false
+ else
+ // Collapse multiple lines into one.
+ this eatLines = true
+ end
+
+ else
+ // A line after any other token is significant.
+ this eatLines = false
+ end
+ end
+
+ if done then return token
+ end
+end
+
+def (this is Morpher) iterate
+ LexerIterator new(lexer: this)
+end
Please sign in to comment.
Something went wrong with that request. Please try again.