Permalink
Browse files

fixing scanner states

  • Loading branch information...
1 parent 147de6c commit e2ce4152e9d7cad58a98e8cdef27b71ca360accc @tenderlove committed Dec 14, 2010
Showing with 32 additions and 3 deletions.
  1. +6 −1 CHANGELOG.rdoc
  2. +10 −2 lib/rexical/generator.rb
  3. +16 −0 test/test_generator.rb
View
7 CHANGELOG.rdoc
@@ -1,4 +1,9 @@
-=== 1.0.5 / Not released
+=== 1.0.6
+
+* Bug fixes
+
+ * scanner states work better. Thanks Mat.
+=== 1.0.5
* Bug fixes
View
12 lib/rexical/generator.rb
@@ -413,8 +413,16 @@ def _next_token
REX_EOT
exclusive_states.each do |es|
+ if es.nil?
+ f.printf <<-REX_EOT
+ when #{(["nil"] + rules.collect{ |rule| rule[1].nil? ? "nil" : rule[1] }).uniq.join(', ')}
+ REX_EOT
+ else
+ f.printf <<-REX_EOT
+ when #{es}
+ REX_EOT
+ end
f.printf <<-REX_EOT
- when #{es ? es.to_s : "nil"}
case
REX_EOT
rules.each do |rule|
@@ -438,7 +446,7 @@ def _next_token
else
if start_state
f.print <<-REX_EOT
- when (state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag}))
+ when (@state == #{start_state}) && (text = @ss.scan(/#{rule_expr}/#{flag}))
;
REX_EOT
View
16 test/test_generator.rb
@@ -71,6 +71,22 @@ class Calculator < Foo::Bar
assert_match 'Calculator < Foo::Bar', source
end
+ def test_stateful_lexer
+ m = build_lexer %q{
+class Foo
+rule
+ \d { @state = :digit; [:foo, text] }
+ :digit \w { @state = nil; [:w, text] }
+end
+ }
+ scanner = m::Foo.new
+ scanner.scan_setup('1w1')
+ assert_tokens [
+ [:foo, '1'],
+ [:w, 'w'],
+ [:foo, '1']], scanner
+ end
+
def test_simple_scanner
m = build_lexer %q{
class Calculator

0 comments on commit e2ce415

Please sign in to comment.