Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

fixing scanner states

  • Loading branch information...
commit e2ce4152e9d7cad58a98e8cdef27b71ca360accc 1 parent 147de6c
Aaron Patterson authored
7 CHANGELOG.rdoc
Source Rendered
... ... @@ -1,4 +1,9 @@
1   -=== 1.0.5 / Not released
  1 +=== 1.0.6
  2 +
  3 +* Bug fixes
  4 +
  5 + * scanner states work better. Thanks Mat.
  6 +=== 1.0.5
2 7
3 8 * Bug fixes
4 9
12 lib/rexical/generator.rb
@@ -413,8 +413,16 @@ def _next_token
413 413 REX_EOT
414 414
415 415 exclusive_states.each do |es|
  416 + if es.nil?
  417 + f.printf <<-REX_EOT
  418 + when #{(["nil"] + rules.collect{ |rule| rule[1].nil? ? "nil" : rule[1] }).uniq.join(', ')}
  419 + REX_EOT
  420 + else
  421 + f.printf <<-REX_EOT
  422 + when #{es}
  423 + REX_EOT
  424 + end
416 425 f.printf <<-REX_EOT
417   - when #{es ? es.to_s : "nil"}
418 426 case
419 427 REX_EOT
420 428 rules.each do |rule|
@@ -438,7 +446,7 @@ def _next_token
438 446 else
439 447 if start_state
440 448 f.print <<-REX_EOT
441   - when (state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag}))
  449 + when (@state == #{start_state}) && (text = @ss.scan(/#{rule_expr}/#{flag}))
442 450 ;
443 451
444 452 REX_EOT
16 test/test_generator.rb
@@ -71,6 +71,22 @@ class Calculator < Foo::Bar
71 71 assert_match 'Calculator < Foo::Bar', source
72 72 end
73 73
  74 + def test_stateful_lexer
  75 + m = build_lexer %q{
  76 +class Foo
  77 +rule
  78 + \d { @state = :digit; [:foo, text] }
  79 + :digit \w { @state = nil; [:w, text] }
  80 +end
  81 + }
  82 + scanner = m::Foo.new
  83 + scanner.scan_setup('1w1')
  84 + assert_tokens [
  85 + [:foo, '1'],
  86 + [:w, 'w'],
  87 + [:foo, '1']], scanner
  88 + end
  89 +
74 90 def test_simple_scanner
75 91 m = build_lexer %q{
76 92 class Calculator

0 comments on commit e2ce415

Please sign in to comment.
Something went wrong with that request. Please try again.