Browse files

Implement the lexer grammar.

  • Loading branch information...
1 parent 706715a commit 40e0c8a15b50fb432dbfcd0041073fc71cd8e6d3 @macournoyer macournoyer committed Mar 27, 2012
Showing with 36 additions and 8 deletions.
  1. +8 −8 lexer.rb
  2. +28 −0 tokens.rex
View
16 lexer.rb
@@ -83,6 +83,9 @@ def _next_token
when (text = @ss.scan(/if/))
action { [:IF, text] }
+ when (text = @ss.scan(/else/))
+ action { [:ELSE, text] }
+
when (text = @ss.scan(/true/))
action { [:TRUE, text] }
@@ -92,14 +95,11 @@ def _next_token
when (text = @ss.scan(/nil/))
action { [:NIL, text] }
- when (text = @ss.scan(/[A-Z][\w_]*/))
- action { [:CONSTANT, text] }
-
- when (text = @ss.scan(/[a-z][\w_]*/))
- action { [:IDENTIFIER, text ]}
+ when (text = @ss.scan(/[a-z]\w*/))
+ action { [:IDENTIFIER, text] }
- when (text = @ss.scan(/\#.*$/))
- ;
+ when (text = @ss.scan(/[A-Z]\w*/))
+ action { [:CONSTANT, text] }
when (text = @ss.scan(/\|\|/))
action { [text, text] }
@@ -120,7 +120,7 @@ def _next_token
action { [text, text] }
when (text = @ss.scan(/./))
- action { [text, text] }
+ action { [text, text] } # ["+", "+"]
else
text = @ss.string[@ss.pos .. -1]
View
28 tokens.rex
@@ -9,7 +9,35 @@ rule
# Whitespace
{BLANK} # no action
+ # Literals
+ \d+ { [:NUMBER, text.to_i] }
+ \"[^"]*\" { [:STRING, text[1..-2]] }
+ \n+ { [:NEWLINE, text] }
+ # Keywords
+ end { [:END, text] }
+ def { [:DEF, text] }
+ class { [:CLASS, text] }
+ if { [:IF, text] }
+ else { [:ELSE, text] }
+ true { [:TRUE, text] }
+ false { [:FALSE, text] }
+ nil { [:NIL, text] }
+
+ # Identifiers
+ [a-z]\w* { [:IDENTIFIER, text] }
+ [A-Z]\w* { [:CONSTANT, text] }
+
+ # Special operators
+ \|\| { [text, text] }
+ && { [text, text] }
+ == { [text, text] }
+ != { [text, text] }
+ >= { [text, text] }
+ <= { [text, text] }
+
+ # Catch all
+ . { [text, text] } # ["+", "+"]
inner
def run(code)

0 comments on commit 40e0c8a

Please sign in to comment.