Added syntax highlighting for verbatim sections that (probably) conta…

…in ruby.

Moved token stream HTML markup out of RDoc::AnyMethod#markup_code into RDoc::TokenStream::to_html

RDoc::RubyLex raises RDoc::RubyLex::Error instead of RDoc::Error.

Fixed generation of rd parsers.

History.txt

@@ -36,6 +36,8 @@
   * RDoc now looks for a <tt>format: parser_name</tt> magic comment to choose
     alternate documentation formats.  The format comment must be in the first
     three lines of the file (to allow for a shebang or modeline).
+  * RDoc makes an effort to syntax-highlight ruby code in verbatim sections.
+    See RDoc::Markup@Paragraphs+and+Verbatim
   * Added RDoc::TopLevel#text? and RDoc::Parser::Text to indicate a
     parsed file contains no ruby constructs.
   * Added <tt>rdoc-label</tt> link scheme which allows bidirectional links.
@@ -47,6 +49,8 @@
     munging.
   * RDoc::RDoc::current is set for the entire RDoc run.
   * Split rdoc/markup/inline into individual files for its component classes.
+  * Moved token stream HTML markup out of RDoc::AnyMethod#markup_code into
+    RDoc::TokenStream::to_html
 
 * Bug fixes
   * Markup defined by RDoc::Markup#add_special inside a <tt><tt></tt> is no

Manifest.txt

@@ -212,6 +212,7 @@ test/test_rdoc_single_class.rb
 test/test_rdoc_stats.rb
 test/test_rdoc_task.rb
 test/test_rdoc_text.rb
+test/test_rdoc_token_stream.rb
 test/test_rdoc_tom_doc.rb
 test/test_rdoc_top_level.rb
 test/xref_data.rb

Rakefile

@@ -40,7 +40,7 @@ Depending on your version of ruby, you may need to install ruby rdoc/ri data:
   self.clean_globs += PARSER_FILES
 
   require_ruby_version '>= 1.8.7'
-  extra_dev_deps << ['racc',     '>= 0']
+  extra_dev_deps << ['racc',     '~> 1.4']
   extra_dev_deps << ['minitest', '~> 2']
   extra_dev_deps << ['ZenTest',  '~> 4']
 
@@ -52,9 +52,9 @@ end
 task 'generate' => PARSER_FILES
 
 rule '.rb' => '.ry' do |t|
-  racc = File.join Gem.dir, 'bin', 'racc'
+  racc = Gem.bin_path 'racc', 'racc'
 
-  sh "#{racc} -l -o #{t.name} #{t.source}"
+  ruby "-rubygems #{racc} -l -o #{t.name} #{t.source}"
 end
 
 # These tasks expect to have the following directory structure:

lib/rdoc/generator/markup.rb

@@ -112,32 +112,7 @@ def add_line_numbers(src)
   def markup_code
     return '' unless @token_stream
 
-    src = ""
-
-    @token_stream.each do |t|
-      next unless t
-
-      style = case t
-              when RDoc::RubyToken::TkCONSTANT then 'ruby-constant'
-              when RDoc::RubyToken::TkKW       then 'ruby-keyword'
-              when RDoc::RubyToken::TkIVAR     then 'ruby-ivar'
-              when RDoc::RubyToken::TkOp       then 'ruby-operator'
-              when RDoc::RubyToken::TkId       then 'ruby-identifier'
-              when RDoc::RubyToken::TkNode     then 'ruby-node'
-              when RDoc::RubyToken::TkCOMMENT  then 'ruby-comment'
-              when RDoc::RubyToken::TkREGEXP   then 'ruby-regexp'
-              when RDoc::RubyToken::TkSTRING   then 'ruby-string'
-              when RDoc::RubyToken::TkVal      then 'ruby-value'
-              end
-
-      text = CGI.escapeHTML t.text
-
-      if style then
-        src << "<span class=\"#{style}\">#{text}</span>"
-      else
-        src << text
-      end
-    end
+    src = RDoc::TokenStream.to_html @token_stream
 
     # dedent the source
     indent = src.length

lib/rdoc/generator/template/darkfish/rdoc.css

@@ -474,13 +474,26 @@ ul.link-list .type {
 
 /* @group Ruby keyword styles */
 
+.description pre,
+.method-description pre {
+  overflow: auto;
+  background: #262626;
+  color: #efefef;
+  border: 1px dashed #999;
+  padding: 0.5em;
+}
+
+.description pre {
+  margin: 0 0.4em;
+}
+
 .ruby-constant   { color: #7fffd4; background: transparent; }
 .ruby-keyword    { color: #00ffff; background: transparent; }
 .ruby-ivar       { color: #eedd82; background: transparent; }
 .ruby-operator   { color: #00ffee; background: transparent; }
 .ruby-identifier { color: #ffdead; background: transparent; }
 .ruby-node       { color: #ffa07a; background: transparent; }
-.ruby-comment    { color: #b22222; font-weight: bold; background: transparent; }
+.ruby-comment    { color: #dc0000; font-weight: bold; background: transparent; }
 .ruby-regexp     { color: #ffa07a; background: transparent; }
 .ruby-value      { color: #7fffd4; background: transparent; }
 

lib/rdoc/markup.rb

@@ -93,6 +93,11 @@
 # have been removed. In addition, the verbatim text has been shifted
 # left, so the amount of indentation of verbatim text is unimportant.
 #
+# For HTML output RDoc makes a small effort to determine if a verbatim section
+# contains ruby source code.  If so, the verbatim block will be marked up as
+# HTML.  Triggers include "def", "class", "module", "require", the "hash
+# rocket"# (=>) or a block call with a parameter.
+#
 # === Headers
 #
 # A line starting with an equal sign (=) is treated as a
@@ -266,7 +271,6 @@
 #   verbatim text outside of the list (the list is therefore closed)
 # regular paragraph after the list
 #
-#
 # == Text Markup
 #
 # === Bold, Italic, Typewriter Text

lib/rdoc/markup/to_html.rb

@@ -187,10 +187,24 @@ def accept_paragraph(paragraph)
   ##
   # Adds +verbatim+ to the output
 
-  def accept_verbatim(verbatim)
-    @res << "\n<pre>"
-    @res << CGI.escapeHTML(verbatim.text.rstrip)
-    @res << "</pre>\n"
+  def accept_verbatim verbatim
+    text = verbatim.text.rstrip
+
+    @res << if parseable? text then
+              options = RDoc::RDoc.current.options if RDoc::RDoc.current
+
+              begin
+                tokens = RDoc::RubyLex.tokenize text, options
+
+                "\n<pre class=\"ruby\">" \
+                "#{RDoc::TokenStream.to_html tokens}" \
+                "</pre>\n"
+              rescue RDoc::RubyLex::Error
+                "\n<pre>#{CGI.escapeHTML text}</pre>\n"
+              end
+            else
+              "\n<pre>#{CGI.escapeHTML text}</pre>\n"
+            end
   end
 
   ##
@@ -361,6 +375,13 @@ def list_end_for(list_type)
     end
   end
 
+  ##
+  # Returns true if Ripper is available it can create a sexp from +text+
+
+  def parseable? text
+    text =~ /\b(def|class|module|require)\b|=>|\{\s?\||do \|/
+  end
+
   ##
   # Converts +item+ to HTML using RDoc::Text#to_html
 

lib/rdoc/ruby_lex.rb

@@ -22,6 +22,12 @@ class RDoc::RubyLex
 
   # :stopdoc:
 
+  ##
+  # Raised upon invalid input
+
+  class Error < RDoc::Error
+  end
+
   extend Exception2MessageMapper
 
   def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
@@ -50,6 +56,19 @@ def self.debug?
 
   self.debug_level = 0
 
+  def self.tokenize text, options
+    tokens = []
+
+    scanner = RDoc::RubyLex.new text, options
+    scanner.exception_on_syntax_error = true
+
+    while token = scanner.token do
+      tokens << token 
+    end
+
+    tokens
+  end
+
   def initialize(content, options)
     lex_init
 
@@ -322,7 +341,7 @@ def token
         tk = @OP.match(self)
         @space_seen = tk.kind_of?(TkSPACE)
       rescue SyntaxError => e
-        raise RDoc::Error, "syntax error: #{e.message}" if
+        raise Error, "syntax error: #{e.message}" if
           @exception_on_syntax_error
 
         tk = TkError.new(@seek, @line_no, @char_no)
@@ -1004,7 +1023,7 @@ def identify_quotation
     elsif ch =~ /\W/
       lt = "\""
     else
-      raise RDoc::Error, "unknown type of %string #{ch.inspect}"
+      raise Error, "unknown type of %string #{ch.inspect}"
     end
     #     if ch !~ /\W/
     #       ungetc
@@ -1039,7 +1058,7 @@ def identify_number(op = "")
       when /[0-7]/
         match = /[0-7_]/
       when /[89]/
-        raise RDoc::Error, "Illegal octal digit"
+        raise Error, "Illegal octal digit"
       else
         return Token(TkINTEGER, num)
       end
@@ -1053,7 +1072,7 @@ def identify_number(op = "")
         if match =~ ch
           if ch == "_"
             if non_digit
-              raise RDoc::Error, "trailing `#{ch}' in number"
+              raise Error, "trailing `#{ch}' in number"
             else
               non_digit = ch
             end
@@ -1065,10 +1084,10 @@ def identify_number(op = "")
           ungetc
           num[-1, 1] = ''
           if len0
-            raise RDoc::Error, "numeric literal without digits"
+            raise Error, "numeric literal without digits"
           end
           if non_digit
-            raise RDoc::Error, "trailing `#{non_digit}' in number"
+            raise Error, "trailing `#{non_digit}' in number"
           end
           break
         end
@@ -1089,7 +1108,7 @@ def identify_number(op = "")
         non_digit = ch
       when allow_point && "."
         if non_digit
-          raise RDoc::Error, "trailing `#{non_digit}' in number"
+          raise Error, "trailing `#{non_digit}' in number"
         end
         type = TkFLOAT
         if peek(0) !~ /[0-9]/
@@ -1101,7 +1120,7 @@ def identify_number(op = "")
         allow_point = false
       when allow_e && "e", allow_e && "E"
         if non_digit
-          raise RDoc::Error, "trailing `#{non_digit}' in number"
+          raise Error, "trailing `#{non_digit}' in number"
         end
         type = TkFLOAT
         if peek(0) =~ /[+-]/
@@ -1112,7 +1131,7 @@ def identify_number(op = "")
         non_digit = ch
       else
         if non_digit
-          raise RDoc::Error, "trailing `#{non_digit}' in number"
+          raise Error, "trailing `#{non_digit}' in number"
         end
         ungetc
         num[-1, 1] = ''

lib/rdoc/ruby_token.rb

@@ -60,6 +60,11 @@ def set_text(text)
       self
     end
 
+    def inspect # :nodoc:
+      klass = self.class.name.split('::').last
+      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @text]
+    end
+
   end
 
   class TkNode < Token
@@ -83,6 +88,12 @@ def set_text text
     end
 
     alias text node
+
+    def inspect # :nodoc:
+      klass = self.class.name.split('::').last
+      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @node]
+    end
+
   end
 
   class TkId < Token
@@ -105,6 +116,12 @@ def set_text text
     end
 
     alias text name
+
+    def inspect # :nodoc:
+      klass = self.class.name.split('::').last
+      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
+    end
+
   end
 
   class TkKW < TkId
@@ -130,6 +147,12 @@ def set_text text
     end
 
     alias text value
+
+    def inspect # :nodoc:
+      klass = self.class.name.split('::').last
+      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @value]
+    end
+
   end
 
   class TkOp < Token
@@ -153,6 +176,12 @@ def set_text text
     end
 
     alias text name
+
+    def inspect # :nodoc:
+      klass = self.class.name.split('::').last
+      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
+    end
+
   end
 
   class TkOPASGN < TkOp
@@ -175,6 +204,12 @@ def ==(other)
     def text
       @text ||= "#{TkToken2Reading[op]}="
     end
+
+    def inspect # :nodoc:
+      klass = self.class.name.split('::').last
+      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @op]
+    end
+
   end
 
   class TkUnknownChar < Token
@@ -197,6 +232,12 @@ def set_text text
     end
 
     alias text name
+
+    def inspect # :nodoc:
+      klass = self.class.name.split('::').last
+      "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
+    end
+
   end
 
   class TkError < Token