Skip to content

Commit

Permalink
Add support for syntax highlighting with new RubyParser AST
Browse files Browse the repository at this point in the history
  • Loading branch information
@lsegal
lsegal committed May 20, 2009
1 parent 413bb1e commit 379227a
Show file tree
Hide file tree
Showing 6 changed files with 99 additions and 40 deletions.
4 changes: 2 additions & 2 deletions README.markdown
View file
Expand Up @@ -87,8 +87,8 @@ following Yardoc'd method:

def reverse(contents)
contents = contents.read if respond_to? :read
contents.reverse
end

end
With the above @param tag, we learn that the contents parameter can either be
a String or any object that responds to the 'read' method, which is more

powerful than the textual description, which says it should be an IO object.

Expand Down
18 changes: 10 additions & 8 deletions lib/yard/autoload.rb
View file
Expand Up @@ -31,12 +31,13 @@ module CodeObjects

module Generators
module Helpers
autoload :BaseHelper, 'yard/generators/helpers/base_helper'
autoload :FilterHelper, 'yard/generators/helpers/filter_helper'
autoload :HtmlHelper, 'yard/generators/helpers/html_helper'
autoload :MarkupHelper, 'yard/generators/helpers/markup_helper'
autoload :MethodHelper, 'yard/generators/helpers/method_helper'
autoload :UMLHelper, 'yard/generators/helpers/uml_helper'
autoload :BaseHelper, 'yard/generators/helpers/base_helper'
autoload :FilterHelper, 'yard/generators/helpers/filter_helper'
autoload :HtmlHelper, 'yard/generators/helpers/html_helper'
autoload :HtmlSyntaxHighlightHelper, 'yard/generators/helpers/html_syntax_highlight_helper'
autoload :MarkupHelper, 'yard/generators/helpers/markup_helper'
autoload :MethodHelper, 'yard/generators/helpers/method_helper'
autoload :UMLHelper, 'yard/generators/helpers/uml_helper'
end

autoload :AttributesGenerator, 'yard/generators/attributes_generator'
Expand Down Expand Up @@ -109,8 +110,9 @@ module RubyToken
end
end

autoload :AstNode, 'yard/parser/ruby/ast_node'
autoload :RubyParser, 'yard/parser/ruby/ruby_parser'
autoload :AstNode, 'yard/parser/ruby/ast_node'
autoload :ParserSyntaxError, 'yard/parser/ruby/ruby_parser'
autoload :RubyParser, 'yard/parser/ruby/ruby_parser'
end

autoload :SourceParser, 'yard/parser/source_parser'
Expand Down
19 changes: 1 addition & 18 deletions lib/yard/generators/helpers/html_helper.rb
View file
Expand Up @@ -4,6 +4,7 @@ module YARD
module Generators::Helpers
module HtmlHelper
include MarkupHelper
include HtmlSyntaxHighlightHelper

SimpleMarkupHtml = RUBY19 ? RDoc::Markup::ToHtml.new : SM::ToHtml.new

Expand Down Expand Up @@ -180,24 +181,6 @@ def url_for_file(filename, anchor = nil)
link = File.relative_path(from, filename)
link + '.html' + (anchor ? '#' + anchor : '')
end

def html_syntax_highlight(source)
tokenlist = Parser::Ruby::Legacy::TokenList.new(source)
tokenlist.map do |s|
prettyclass = s.class.class_name.sub(/^Tk/, '').downcase
prettysuper = s.class.superclass.class_name.sub(/^Tk/, '').downcase

case s
when Parser::Ruby::Legacy::RubyToken::TkWhitespace, Parser::Ruby::Legacy::RubyToken::TkUnknownChar
h s.text
when Parser::Ruby::Legacy::RubyToken::TkId
prettyval = h(s.text)
"<span class='#{prettyval} #{prettyclass} #{prettysuper}'>#{prettyval}</span>"
else
"<span class='#{prettyclass} #{prettysuper}'>#{h s.text}</span>"
end
end.join
end
end
end
end
Expand Down
49 changes: 49 additions & 0 deletions lib/yard/generators/helpers/html_syntax_highlight_helper.rb
View file
@@ -0,0 +1,49 @@
module YARD
module Generators
module Helpers
module HtmlSyntaxHighlightHelper
def html_syntax_highlight(source)
tokenlist = Parser::Ruby::RubyParser.parse(source, "(syntax_highlight)").tokens
output = ""
tokenlist.each do |s|
output << "<span class='tstring'>" if [:tstring_beg, :regexp_beg].include?(s[0])
case s.first
when :nl, :ignored_nl, :sp
output << h(s.last)
when :ident
output << "<span class='id #{h(s.last)}'>#{h(s.last)}</span>"
else
output << "<span class='#{s.first}'>#{h(s.last)}</span>"
end
output << "</span>" if [:tstring_end, :regexp_end].include?(s[0])
end
output
rescue Parser::Ruby::ParserSyntaxError
source
end

def html_syntax_highlight_legacy(source)
tokenlist = Parser::Ruby::Legacy::TokenList.new(source)
tokenlist.map do |s|
prettyclass = s.class.class_name.sub(/^Tk/, '').downcase
prettysuper = s.class.superclass.class_name.sub(/^Tk/, '').downcase

case s
when Parser::Ruby::Legacy::RubyToken::TkWhitespace, Parser::Ruby::Legacy::RubyToken::TkUnknownChar
h s.text
when Parser::Ruby::Legacy::RubyToken::TkId
prettyval = h(s.text)
"<span class='#{prettyval} #{prettyclass} #{prettysuper}'>#{prettyval}</span>"
else
"<span class='#{prettyclass} #{prettysuper}'>#{h s.text}</span>"
end
end.join
end

if RUBY18
alias html_syntax_highlight html_syntax_highlight_legacy
end
end
end
end
end
38 changes: 30 additions & 8 deletions lib/yard/parser/ruby/ruby_parser.rb
View file
Expand Up @@ -3,6 +3,8 @@
module YARD
module Parser
module Ruby
class ParserSyntaxError < SyntaxError; end

class RubyParser < Ripper
class << self
def no_comment(*toks)
Expand All @@ -18,25 +20,22 @@ def no_comment(*toks)
:void_stmt, :stmt_body, :var_ref, :args, :self, :string_embexpr,
:string_dvar, :xstring_literal, :rest_param, :blockarg

attr_reader :ast, :charno, :comments, :file
attr_reader :ast, :charno, :comments, :file, :tokens

def initialize(source, filename, *args)
super
@file = filename
@source = source
@tokens = []
@comments = []
@charno = 0
end

def tokens
@source
end

def parse
@ast = super
@ast.insert_comments(@comments)
@ast.full_source = @source
@ast.file = @file
insert_comments
self
end

Expand Down Expand Up @@ -86,8 +85,17 @@ def on_#{event}(tok)
def visit_token(token, data)
ch = charno
@charno += data.length
add_token(token, data)
AstNode.new(token, [data], line: lineno, char: ch, token: true)
end

def add_token(token, data)
if @tokens.last && @tokens.last[0] == :symbeg
@tokens[-1] = [:symbol, ":" + data]
else
@tokens << [token, data]
end
end

def on_program(*args)
args.first
Expand Down Expand Up @@ -118,15 +126,29 @@ def on_comment(comment)
end

if append_comment
@comments.last.first.push(comment[1..-1])
@comments.last.first.push(comment.gsub(/^\#{1,2}\s{0,1}/, '').chomp)
@comments.last[-1] = lineno
else
@comments << [[comment[1..-1]], lineno]
end
end

def on_parse_error(msg)
raise SyntaxError, "in `#{@file}`:(#{lineno},#{column}): #{msg}"
raise ParserSyntaxError, "in `#{@file}`:(#{lineno},#{column}): #{msg}"
end

def insert_comments
comments = @comments.dup
@ast.traverse do |node|
comments.each.with_index do |c, i|
next if c.empty? || node.line.nil?
if node.line.between?(c.last, c.last + 2)
comments.delete_at(i)
node.docstring = c.first.join("\n")
break
end
end
end
end
end
end
Expand Down
11 changes: 7 additions & 4 deletions templates/default/fulldoc/html/syntax_highlight.css
View file
Expand Up @@ -7,15 +7,18 @@
pre.code, pre.lines { font-family: Monaco, Courier, monospace; }
pre.code { color: #fff; }
pre.code .info.file { color: #888; }
pre.code .val { color: #61CE3C; }
pre.code .dstring { color: #61DE3C; }
pre.code .val, pre.code .int, pre.code .float { color: #61CE3C; }
pre.code .dstring, pre.code .tstring_content,
pre.code .regexp_beg, pre.code .regexp_end,
pre.code .tstring_beg, pre.code .tstring_end { color: #61DE3C; }
pre.code .embexpr_beg, pre.code .tstring, pre.code .tstring .rbrace { color: #7CFE65; }
pre.code .fid, pre.code .id.new, pre.code .id.to_s,
pre.code .id.to_sym, pre.code .id.to_f,
pre.code .dot + pre.code .id,
pre.code .id.to_i pre.code .id.each { color: #FF6400; }
pre.code .comment { color: #AEAEAE; }
pre.code .constant, pre.code .symbol { color: #D8FA3C; }
pre.code .constant, pre.code .const, pre.code .symbol, pre.code .symbeg { color: #D8FA3C; }
pre.code .kw { color: #FBDE2D; }
pre.code .ivar { color: #AACEFB; }
pre.code .gvar, pre.code .id.nth_ref { color: #BADEFF; }
pre.code .gvar, pre.code .backref, pre.code .id.nth_ref { color: #BADEFF; }
pre.code .regexp, .dregexp { color: #EA5EFB; }

0 comments on commit 379227a

Please sign in to comment.