Fixed tokenization of "a%1"

Updated failed tokenization examples to use bad octal digits Conflicts: History.rdoc test/test_rdoc_markup_to_html.rb test/test_rdoc_markup_to_html_snippet.rb
ruby · Feb 24, 2013 · 976d31f · 976d31f
1 parent 8eb2ae2
commit 976d31f
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 4 deletions.
diff --git a/History.rdoc b/History.rdoc
@@ -5,6 +5,7 @@
     Ruby Bug #6488 by Benny Lyne Amorsen.
   * Fixed lexing of character syntax (<code>?x</code>).  Reported by Xavier
     Noria.
+  * Fixed tokenization of % when it is not followed by a $-string type
 
 === 3.12.1 / 2013-02-05
 

diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
@@ -1034,7 +1034,7 @@ def identify_quotation
       type = nil
       lt = "\""
     else
-      raise Error, "unknown type of %string #{type.inspect}"
+      return Token(TkMOD, '%')
     end
     #     if ch !~ /\W/
     #       ungetc

diff --git a/test/test_rdoc_markup_to_html_snippet.rb b/test/test_rdoc_markup_to_html_snippet.rb
@@ -379,19 +379,21 @@ def test_accept_verbatim_ruby_error
     rdoc.options = options
     RDoc::RDoc.current = rdoc
 
-    verb = @RM::Verbatim.new("a %z'foo' # => blah\n")
+    verb = @RM::Verbatim.new("a % 09 # => blah\n")
 
     @to.start_accepting
     @to.accept_verbatim verb
 
+    inner = CGI.escapeHTML "a % 09 # => blah"
+
     expected = <<-EXPECTED
 
-<pre>a %z'foo' # =&gt; blah
+<pre>a % 09 # =&gt; blah
 </pre>
     EXPECTED
 
     assert_equal expected, @to.res.join
-    assert_equal 19, @to.characters
+    assert_equal 16, @to.characters
   end
 
   def test_add_paragraph

diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb
@@ -105,6 +105,21 @@ def test_class_tokenize_heredoc_percent_N
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_percent_1
+    tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new(0, 1, 0, 'v'),
+      @TK::TkMOD.new(       1, 1, 1, '%'),
+      @TK::TkINTEGER.new(   2, 1, 2, '10'),
+      @TK::TkEQ.new(        4, 1, 4, '=='),
+      @TK::TkINTEGER.new(   6, 1, 6, '10'),
+      @TK::TkNL.new(        8, 1, 8, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_percent_r
     tokens = RDoc::RubyLex.tokenize '%r[hi]', nil