Merge remote-tracking branch 'upstream/master' into fix-for-symbol-de…

…tection-when-nested-method

lib/rdoc/ruby_lex.rb

@@ -112,6 +112,7 @@ def initialize(content, options)
     @indent_stack = []
     @lex_state = :EXPR_BEG
     @space_seen = false
+    @after_question = false
 
     @continue = false
     @line = ""
@@ -383,6 +384,8 @@ def token
       set_token_position tk.seek, tk.line_no, tk.char_no
       tk = Token(tk1.class, tk.text + tk1.text)
     end
+    @after_question = false if @after_question and !(TkQUESTION === tk)
+
     #      Tracer.off
     tk
   end
@@ -454,15 +457,18 @@ def lex_init()
                  proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
       |op, io|
       @ltype = "="
-      res = ''
-      nil until getc == "\n"
+      res = op
+      until (ch = getc) == "\n" do
+        res << ch
+      end
+      res << ch
 
       until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do
         (ch = getc)
         res << ch
       end
 
-      gets # consume =end
+      res << gets # consume =end
 
       @ltype = nil
       Token(TkRD_COMMENT, res)
@@ -597,6 +603,7 @@ def lex_init()
       |op, io|
       if @lex_state == :EXPR_END
         @lex_state = :EXPR_BEG
+        @after_question = true
         Token(TkQUESTION)
       else
         ch = getc
@@ -606,6 +613,7 @@ def lex_init()
           Token(TkQUESTION)
         else
           @lex_state = :EXPR_END
+          ch << getc if "\\" == ch
           Token(TkCHAR, "?#{ch}")
         end
       end
@@ -1364,7 +1372,10 @@ def identify_string(ltype, quoted = ltype, type = nil)
         end
       end
 
-      if subtype
+      if peek(0) == ':' and !peek_match?(/^::/) and :EXPR_BEG == @lex_state and !@after_question
+        str.concat getc
+        return Token(TkSYMBOL, str)
+      elsif subtype
         Token(DLtype2Token[ltype], str)
       else
         Token(Ltype2Token[ltype], str)

lib/rdoc/ruby_token.rb

@@ -322,6 +322,7 @@ def Token(token, value = nil)
     [:TklEND,       TkKW,  "END",      :EXPR_END],
     [:Tk__LINE__,   TkKW,  "__LINE__", :EXPR_END],
     [:Tk__FILE__,   TkKW,  "__FILE__", :EXPR_END],
+    [:Tk__ENCODING__,TkKW, "__ENCODING__", :EXPR_END],
 
     [:TkIDENTIFIER, TkId],
     [:TkFID,        TkId],

lib/rdoc/token_stream.rb

@@ -44,10 +44,18 @@ def self.to_html token_stream
               when RDoc::RubyToken::TkVal      then 'ruby-value'
               end
 
-      text = CGI.escapeHTML t.text
+      comment_with_nl = false
+      case t
+      when RDoc::RubyToken::TkRD_COMMENT, RDoc::RubyToken::TkHEREDOCEND
+        comment_with_nl = true if t.text =~ /\n$/
+        text = t.text.rstrip
+      else
+        text = t.text
+      end
+      text = CGI.escapeHTML text
 
       if style then
-        "<span class=\"#{style}\">#{text}</span>"
+        "<span class=\"#{style}\">#{text}</span>#{"\n" if comment_with_nl}"
       else
         text
       end

test/test_rdoc_parser_ruby.rb

@@ -74,7 +74,7 @@ class C; end
 
     comment = parser.collect_first_comment
 
-    assert_equal RDoc::Comment.new("first\n\n", @top_level), comment
+    assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n\n", @top_level), comment
   end
 
   def test_get_class_or_module
@@ -2513,8 +2513,8 @@ def blah()
     expected = <<EXPTECTED
   <span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>()
     <span class="ruby-identifier">&lt;&lt;~EOM</span> <span class="ruby-keyword">if</span> <span class="ruby-keyword">true</span>
-<span class="ruby-value"></span><span class="ruby-identifier">    EOM
-</span>  <span class="ruby-keyword">end</span>
+<span class="ruby-value"></span><span class="ruby-identifier">    EOM</span>
+  <span class="ruby-keyword">end</span>
 EXPTECTED
     expected = expected.rstrip
 
@@ -2528,6 +2528,40 @@ def blah()
     assert_equal markup_code, expected
   end
 
+  def test_parse_statements_embdoc_in_document
+    @filename = 'file.rb'
+    util_parser <<RUBY
+class Foo
+  # doc
+  #
+  #   =begin
+  #   test embdoc
+  #   =end
+  #
+  def blah
+  end
+end
+RUBY
+
+    expected = <<EXPTECTED
+<p>doc
+
+<pre class="ruby"><span class="ruby-comment">=begin
+test embdoc
+=end</span>
+</pre>
+EXPTECTED
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_comment = blah.search_record[6]
+    assert_equal markup_comment, expected
+  end
+
   def test_parse_require_dynamic_string
     content = <<-RUBY
 prefix = 'path'
@@ -3002,11 +3036,11 @@ def m() end
 
     foo = @top_level.classes.first
 
-    assert_equal 'Foo comment', foo.comment.text
+    assert_equal "=begin rdoc\nFoo comment\n=end", foo.comment.text
 
     m = foo.method_list.first
 
-    assert_equal 'm comment', m.comment.text
+    assert_equal "=begin\nm comment\n=end", m.comment.text
   end
 
   def test_scan_block_comment_nested # Issue #41
@@ -3028,7 +3062,7 @@ class Bar
     foo = @top_level.modules.first
 
     assert_equal 'Foo', foo.full_name
-    assert_equal 'findmeindoc', foo.comment.text
+    assert_equal "=begin rdoc\nfindmeindoc\n=end", foo.comment.text
 
     bar = foo.classes.first
 
@@ -3075,12 +3109,12 @@ def lauren
 
     foo = @top_level.classes.first
 
-    assert_equal "= DESCRIPTION\n\nThis is a simple test class\n\n= RUMPUS\n\nIs a silly word",
+    assert_equal "=begin rdoc\n\n= DESCRIPTION\n\nThis is a simple test class\n\n= RUMPUS\n\nIs a silly word\n\n=end",
       foo.comment.text
 
     m = foo.method_list.first
 
-    assert_equal 'A nice girl', m.comment.text
+    assert_equal "=begin rdoc\nA nice girl\n=end", m.comment.text
   end
 
   def test_scan_class_nested_nodoc

test/test_rdoc_ruby_lex.rb

@@ -62,17 +62,39 @@ def test_class_tokenize___END__
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize___ENCODING__
+    tokens = RDoc::RubyLex.tokenize '__ENCODING__', nil
+
+    expected = [
+      @TK::Tk__ENCODING__.new( 0, 1,  0, '__ENCODING__'),
+      @TK::TkNL          .new(12, 1, 12, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_character_literal
-    tokens = RDoc::RubyLex.tokenize "?\\", nil
+    tokens = RDoc::RubyLex.tokenize "?c", nil
 
     expected = [
-      @TK::TkCHAR.new( 0, 1,  0, "?\\"),
+      @TK::TkCHAR.new( 0, 1,  0, "?c"),
       @TK::TkNL  .new( 2, 1,  2, "\n"),
     ]
 
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_character_literal_with_escape
+    tokens = RDoc::RubyLex.tokenize "?\\s", nil
+
+    expected = [
+      @TK::TkCHAR.new( 0, 1,  0, "?\\s"),
+      @TK::TkNL  .new( 3, 1,  3, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_def_heredoc
     tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
 def x
@@ -831,6 +853,36 @@ def test_class_tokenize_symbol_for_nested_method
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_symbol_with_quote
+    tokens = RDoc::RubyLex.tokenize <<RUBY, nil
+a.include?()?"a":"b"
+{"t":1,'t2':2}
+RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, "a"),
+      @TK::TkDOT       .new( 1, 1,  1, "."),
+      @TK::TkFID       .new( 2, 1,  2, "include?"),
+      @TK::TkLPAREN    .new(10, 1, 10, "("),
+      @TK::TkRPAREN    .new(11, 1, 11, ")"),
+      @TK::TkQUESTION  .new(12, 1, 12, "?"),
+      @TK::TkSTRING    .new(13, 1, 13, "\"a\""),
+      @TK::TkCOLON     .new(16, 1, 16, ":"),
+      @TK::TkSTRING    .new(17, 1, 17, "\"b\""),
+      @TK::TkNL        .new(20, 1, 20, "\n"),
+      @TK::TkLBRACE    .new(21, 2,  0, "{"),
+      @TK::TkSYMBOL    .new(22, 2,  1, "\"t\":"),
+      @TK::TkINTEGER   .new(26, 2,  5, "1"),
+      @TK::TkCOMMA     .new(27, 2,  6, ","),
+      @TK::TkSYMBOL    .new(28, 2,  7, "'t2':"),
+      @TK::TkINTEGER   .new(33, 2, 12, "2"),
+      @TK::TkRBRACE    .new(34, 2, 13, "}"),
+      @TK::TkNL        .new(35, 2, 21, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_unary_minus
     ruby_lex = RDoc::RubyLex.new("-1", nil)
     assert_equal("-1", ruby_lex.token.value)