Permalink
Browse files

Added syntax highlighting for verbatim sections that (probably) conta…

…in ruby.

Moved token stream HTML markup out of RDoc::AnyMethod#markup_code into RDoc::TokenStream::to_html

RDoc::RubyLex raises RDoc::RubyLex::Error instead of RDoc::Error.

Fixed generation of rd parsers.
  • Loading branch information...
1 parent d346ffc commit 0a8902f9db7eeadbe58c2adb377b3620ef9d1d46 @drbrain drbrain committed Sep 15, 2011
View
@@ -36,6 +36,8 @@
* RDoc now looks for a <tt>format: parser_name</tt> magic comment to choose
alternate documentation formats. The format comment must be in the first
three lines of the file (to allow for a shebang or modeline).
+ * RDoc makes an effort to syntax-highlight ruby code in verbatim sections.
+ See RDoc::Markup@Paragraphs+and+Verbatim
* Added RDoc::TopLevel#text? and RDoc::Parser::Text to indicate a
parsed file contains no ruby constructs.
* Added <tt>rdoc-label</tt> link scheme which allows bidirectional links.
@@ -47,6 +49,8 @@
munging.
* RDoc::RDoc::current is set for the entire RDoc run.
* Split rdoc/markup/inline into individual files for its component classes.
+ * Moved token stream HTML markup out of RDoc::AnyMethod#markup_code into
+ RDoc::TokenStream::to_html
* Bug fixes
* Markup defined by RDoc::Markup#add_special inside a <tt><tt></tt> is no
View
@@ -212,6 +212,7 @@ test/test_rdoc_single_class.rb
test/test_rdoc_stats.rb
test/test_rdoc_task.rb
test/test_rdoc_text.rb
+test/test_rdoc_token_stream.rb
test/test_rdoc_tom_doc.rb
test/test_rdoc_top_level.rb
test/xref_data.rb
View
@@ -40,7 +40,7 @@ Depending on your version of ruby, you may need to install ruby rdoc/ri data:
self.clean_globs += PARSER_FILES
require_ruby_version '>= 1.8.7'
- extra_dev_deps << ['racc', '>= 0']
+ extra_dev_deps << ['racc', '~> 1.4']
extra_dev_deps << ['minitest', '~> 2']
extra_dev_deps << ['ZenTest', '~> 4']
@@ -52,9 +52,9 @@ end
task 'generate' => PARSER_FILES
rule '.rb' => '.ry' do |t|
- racc = File.join Gem.dir, 'bin', 'racc'
+ racc = Gem.bin_path 'racc', 'racc'
- sh "#{racc} -l -o #{t.name} #{t.source}"
+ ruby "-rubygems #{racc} -l -o #{t.name} #{t.source}"
end
# These tasks expect to have the following directory structure:
@@ -112,32 +112,7 @@ def add_line_numbers(src)
def markup_code
return '' unless @token_stream
- src = ""
-
- @token_stream.each do |t|
- next unless t
-
- style = case t
- when RDoc::RubyToken::TkCONSTANT then 'ruby-constant'
- when RDoc::RubyToken::TkKW then 'ruby-keyword'
- when RDoc::RubyToken::TkIVAR then 'ruby-ivar'
- when RDoc::RubyToken::TkOp then 'ruby-operator'
- when RDoc::RubyToken::TkId then 'ruby-identifier'
- when RDoc::RubyToken::TkNode then 'ruby-node'
- when RDoc::RubyToken::TkCOMMENT then 'ruby-comment'
- when RDoc::RubyToken::TkREGEXP then 'ruby-regexp'
- when RDoc::RubyToken::TkSTRING then 'ruby-string'
- when RDoc::RubyToken::TkVal then 'ruby-value'
- end
-
- text = CGI.escapeHTML t.text
-
- if style then
- src << "<span class=\"#{style}\">#{text}</span>"
- else
- src << text
- end
- end
+ src = RDoc::TokenStream.to_html @token_stream
# dedent the source
indent = src.length
@@ -474,13 +474,26 @@ ul.link-list .type {
/* @group Ruby keyword styles */
+.description pre,
+.method-description pre {
+ overflow: auto;
+ background: #262626;
+ color: #efefef;
+ border: 1px dashed #999;
+ padding: 0.5em;
+}
+
+.description pre {
+ margin: 0 0.4em;
+}
+
.ruby-constant { color: #7fffd4; background: transparent; }
.ruby-keyword { color: #00ffff; background: transparent; }
.ruby-ivar { color: #eedd82; background: transparent; }
.ruby-operator { color: #00ffee; background: transparent; }
.ruby-identifier { color: #ffdead; background: transparent; }
.ruby-node { color: #ffa07a; background: transparent; }
-.ruby-comment { color: #b22222; font-weight: bold; background: transparent; }
+.ruby-comment { color: #dc0000; font-weight: bold; background: transparent; }
.ruby-regexp { color: #ffa07a; background: transparent; }
.ruby-value { color: #7fffd4; background: transparent; }
View
@@ -93,6 +93,11 @@
# have been removed. In addition, the verbatim text has been shifted
# left, so the amount of indentation of verbatim text is unimportant.
#
+# For HTML output RDoc makes a small effort to determine if a verbatim section
+# contains ruby source code. If so, the verbatim block will be marked up as
+# HTML. Triggers include "def", "class", "module", "require", the "hash
+# rocket"# (=>) or a block call with a parameter.
+#
# === Headers
#
# A line starting with an equal sign (=) is treated as a
@@ -266,7 +271,6 @@
# verbatim text outside of the list (the list is therefore closed)
# regular paragraph after the list
#
-#
# == Text Markup
#
# === Bold, Italic, Typewriter Text
View
@@ -187,10 +187,24 @@ def accept_paragraph(paragraph)
##
# Adds +verbatim+ to the output
- def accept_verbatim(verbatim)
- @res << "\n<pre>"
- @res << CGI.escapeHTML(verbatim.text.rstrip)
- @res << "</pre>\n"
+ def accept_verbatim verbatim
+ text = verbatim.text.rstrip
+
+ @res << if parseable? text then
+ options = RDoc::RDoc.current.options if RDoc::RDoc.current
+
+ begin
+ tokens = RDoc::RubyLex.tokenize text, options
+
+ "\n<pre class=\"ruby\">" \
+ "#{RDoc::TokenStream.to_html tokens}" \
+ "</pre>\n"
+ rescue RDoc::RubyLex::Error
+ "\n<pre>#{CGI.escapeHTML text}</pre>\n"
+ end
+ else
+ "\n<pre>#{CGI.escapeHTML text}</pre>\n"
+ end
end
##
@@ -362,6 +376,13 @@ def list_end_for(list_type)
end
##
+ # Returns true if Ripper is available it can create a sexp from +text+
+
+ def parseable? text
+ text =~ /\b(def|class|module|require)\b|=>|\{\s?\||do \|/
+ end
+
+ ##
# Converts +item+ to HTML using RDoc::Text#to_html
def to_html item
View
@@ -22,6 +22,12 @@ class RDoc::RubyLex
# :stopdoc:
+ ##
+ # Raised upon invalid input
+
+ class Error < RDoc::Error
+ end
+
extend Exception2MessageMapper
def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
@@ -50,6 +56,19 @@ def self.debug?
self.debug_level = 0
+ def self.tokenize text, options
+ tokens = []
+
+ scanner = RDoc::RubyLex.new text, options
+ scanner.exception_on_syntax_error = true
+
+ while token = scanner.token do
+ tokens << token
+ end
+
+ tokens
+ end
+
def initialize(content, options)
lex_init
@@ -322,7 +341,7 @@ def token
tk = @OP.match(self)
@space_seen = tk.kind_of?(TkSPACE)
rescue SyntaxError => e
- raise RDoc::Error, "syntax error: #{e.message}" if
+ raise Error, "syntax error: #{e.message}" if
@exception_on_syntax_error
tk = TkError.new(@seek, @line_no, @char_no)
@@ -1004,7 +1023,7 @@ def identify_quotation
elsif ch =~ /\W/
lt = "\""
else
- raise RDoc::Error, "unknown type of %string #{ch.inspect}"
+ raise Error, "unknown type of %string #{ch.inspect}"
end
# if ch !~ /\W/
# ungetc
@@ -1039,7 +1058,7 @@ def identify_number(op = "")
when /[0-7]/
match = /[0-7_]/
when /[89]/
- raise RDoc::Error, "Illegal octal digit"
+ raise Error, "Illegal octal digit"
else
return Token(TkINTEGER, num)
end
@@ -1053,7 +1072,7 @@ def identify_number(op = "")
if match =~ ch
if ch == "_"
if non_digit
- raise RDoc::Error, "trailing `#{ch}' in number"
+ raise Error, "trailing `#{ch}' in number"
else
non_digit = ch
end
@@ -1065,10 +1084,10 @@ def identify_number(op = "")
ungetc
num[-1, 1] = ''
if len0
- raise RDoc::Error, "numeric literal without digits"
+ raise Error, "numeric literal without digits"
end
if non_digit
- raise RDoc::Error, "trailing `#{non_digit}' in number"
+ raise Error, "trailing `#{non_digit}' in number"
end
break
end
@@ -1089,7 +1108,7 @@ def identify_number(op = "")
non_digit = ch
when allow_point && "."
if non_digit
- raise RDoc::Error, "trailing `#{non_digit}' in number"
+ raise Error, "trailing `#{non_digit}' in number"
end
type = TkFLOAT
if peek(0) !~ /[0-9]/
@@ -1101,7 +1120,7 @@ def identify_number(op = "")
allow_point = false
when allow_e && "e", allow_e && "E"
if non_digit
- raise RDoc::Error, "trailing `#{non_digit}' in number"
+ raise Error, "trailing `#{non_digit}' in number"
end
type = TkFLOAT
if peek(0) =~ /[+-]/
@@ -1112,7 +1131,7 @@ def identify_number(op = "")
non_digit = ch
else
if non_digit
- raise RDoc::Error, "trailing `#{non_digit}' in number"
+ raise Error, "trailing `#{non_digit}' in number"
end
ungetc
num[-1, 1] = ''
View
@@ -60,6 +60,11 @@ def set_text(text)
self
end
+ def inspect # :nodoc:
+ klass = self.class.name.split('::').last
+ "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @text]
+ end
+
end
class TkNode < Token
@@ -83,6 +88,12 @@ def set_text text
end
alias text node
+
+ def inspect # :nodoc:
+ klass = self.class.name.split('::').last
+ "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @node]
+ end
+
end
class TkId < Token
@@ -105,6 +116,12 @@ def set_text text
end
alias text name
+
+ def inspect # :nodoc:
+ klass = self.class.name.split('::').last
+ "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
+ end
+
end
class TkKW < TkId
@@ -130,6 +147,12 @@ def set_text text
end
alias text value
+
+ def inspect # :nodoc:
+ klass = self.class.name.split('::').last
+ "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @value]
+ end
+
end
class TkOp < Token
@@ -153,6 +176,12 @@ def set_text text
end
alias text name
+
+ def inspect # :nodoc:
+ klass = self.class.name.split('::').last
+ "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
+ end
+
end
class TkOPASGN < TkOp
@@ -175,6 +204,12 @@ def ==(other)
def text
@text ||= "#{TkToken2Reading[op]}="
end
+
+ def inspect # :nodoc:
+ klass = self.class.name.split('::').last
+ "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @op]
+ end
+
end
class TkUnknownChar < Token
@@ -197,6 +232,12 @@ def set_text text
end
alias text name
+
+ def inspect # :nodoc:
+ klass = self.class.name.split('::').last
+ "{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
+ end
+
end
class TkError < Token
Oops, something went wrong.

0 comments on commit 0a8902f

Please sign in to comment.