attic/lib/node.rb attic/lib/sanitize.rb attic/test/unit/sanitize_test.rb vendor/plugins/HTML5lib/README vendor/plugins/HTML5lib/Rakefile.rb vendor/plugins/HTML5lib/lib/html5lib.rb vendor/plugins/HTML5lib/lib/html5lib/constants.rb vendor/plugins/HTML5lib/lib/html5lib/html5parser.rb vendor/plugins/HTML5lib/lib/html5lib/inputstream.rb vendor/plugins/HTML5lib/lib/html5lib/liberalxmlparser.rb vendor/plugins/HTML5lib/lib/html5lib/sanitizer.rb vendor/plugins/HTML5lib/lib/html5lib/tokenizer.rb vendor/plugins/HTML5lib/lib/html5lib/treebuilders.rb vendor/plugins/HTML5lib/lib/html5lib/treebuilders/base.rb vendor/plugins/HTML5lib/lib/html5lib/treebuilders/hpricot.rb vendor/plugins/HTML5lib/lib/html5lib/treebuilders/rexml.rb vendor/plugins/HTML5lib/lib/html5lib/treebuilders/simpletree.rb vendor/plugins/HTML5lib/tests/preamble.rb vendor/plugins/HTML5lib/tests/test_encoding.rb vendor/plugins/HTML5lib/tests/test_lxp.rb vendor/plugins/HTML5lib/tests/test_parser.rb vendor/plugins/HTML5lib/tests/test_sanitizer.rb vendor/plugins/HTML5lib/tests/test_tokenizer.rb vendor/plugins/HTML5lib/tests/tokenizer_test_parser.rb @@ -294,13 +294,13 @@ class WikiController < ApplicationController def s5 if @web.markup == :markdownMML - @s5_content = sanitize_html(Maruku.new(@page.content.delete("\r\x01-\x08\x0B\x0C\x0E-\x1F"), + @s5_content = sanitize_xhtml(Maruku.new(@page.content.delete("\r\x01-\x08\x0B\x0C\x0E-\x1F"), {:math_enabled => true, :math_numbered => ['\\[','\\begin{equation}'], :content_only => true, - :author => @page.author, :title => @page.plain_name}).to_s5).to_ncr + :author => @page.author, :title => @page.plain_name}).to_s5.to_ncr) elsif @web.markup == :markdown - @s5_content = sanitize_html(Maruku.new(@page.content.delete("\r\x01-\x08\x0B\x0C\x0E-\x1F"), + @s5_content = sanitize_xhtml(Maruku.new(@page.content.delete("\r\x01-\x08\x0B\x0C\x0E-\x1F"), {:math_enabled => false, :content_only => true, - :author => @page.author, :title => @page.plain_name}).to_s5).to_ncr + :author => @page.author, :title => @page.plain_name}).to_s5.to_ncr) else @s5_content = "S5 not supported with this text filter" end app/controllers/wiki_controller.rb @@ -32,7 +32,7 @@ module Engines redcloth.filter_html = false redcloth.no_span_caps = false html = redcloth.to_html(:textile) - sanitize_html(html) + sanitize_xhtml(html) end end @@ -43,7 +43,7 @@ module Engines require_dependency 'maruku' require_dependency 'maruku/ext/math' html = Maruku.new(@content.delete("\r\x01-\x08\x0B\x0C\x0E-\x1F"), {:math_enabled => false}).to_html - sanitize_html(html).to_ncr + sanitize_xhtml(html.to_ncr) end end @@ -55,7 +55,7 @@ module Engines require_dependency 'maruku/ext/math' html = Maruku.new(@content.delete("\r\x01-\x08\x0B\x0C\x0E-\x1F"), {:math_enabled => true, :math_numbered => ['\\[','\\begin{equation}']}).to_html - sanitize_html(html).to_ncr + sanitize_xhtml(html.to_ncr) end end @@ -68,7 +68,7 @@ module Engines redcloth.filter_html = false redcloth.no_span_caps = false html = redcloth.to_html - sanitize_html(html) + sanitize_xhtml(html) end end @@ -78,7 +78,7 @@ module Engines def mask require_dependency 'rdocsupport' html = RDocSupport::RDocFormatter.new(@content).to_html - sanitize_html(html) + sanitize_xhtml(html) end end lib/chunks/engines.rb @@ -1,207 +1,26 @@ module Sanitize -# This module provides sanitization of XHTML+MathML+SVG +# This module provides sanitization of XHTML+MathML+SVG # and of inline style attributes. # -# Based heavily on Sam Ruby's code in the Universal FeedParser. - - require 'html/tokenizer' - require 'node' - - acceptable_elements = ['a', 'abbr', 'acronym', 'address', 'area', 'b', - 'big', 'blockquote', 'br', 'button', 'caption', 'center', 'cite', - 'code', 'col', 'colgroup', 'dd', 'del', 'dfn', 'dir', 'div', 'dl', 'dt', - 'em', 'fieldset', 'font', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', - 'hr', 'i', 'img', 'input', 'ins', 'kbd', 'label', 'legend', 'li', 'map', - 'menu', 'ol', 'optgroup', 'option', 'p', 'pre', 'q', 's', 'samp', - 'select', 'small', 'span', 'strike', 'strong', 'sub', 'sup', 'table', - 'tbody', 'td', 'textarea', 'tfoot', 'th', 'thead', 'tr', 'tt', 'u', - 'ul', 'var'] - - mathml_elements = ['maction', 'math', 'merror', 'mfrac', 'mi', - 'mmultiscripts', 'mn', 'mo', 'mover', 'mpadded', 'mphantom', - 'mprescripts', 'mroot', 'mrow', 'mspace', 'msqrt', 'mstyle', 'msub', - 'msubsup', 'msup', 'mtable', 'mtd', 'mtext', 'mtr', 'munder', - 'munderover', 'none'] - - svg_elements = ['a', 'animate', 'animateColor', 'animateMotion', - 'animateTransform', 'circle', 'defs', 'desc', 'ellipse', 'font-face', - 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', 'image', - 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', - 'mpath', 'path', 'polygon', 'polyline', 'radialGradient', 'rect', - 'set', 'stop', 'svg', 'switch', 'text', 'title', 'tspan', 'use'] - - acceptable_attributes = ['abbr', 'accept', 'accept-charset', 'accesskey', - 'action', 'align', 'alt', 'axis', 'border', 'cellpadding', - 'cellspacing', 'char', 'charoff', 'charset', 'checked', 'cite', 'class', - 'clear', 'cols', 'colspan', 'color', 'compact', 'coords', 'datetime', - 'dir', 'disabled', 'enctype', 'for', 'frame', 'headers', 'height', - 'href', 'hreflang', 'hspace', 'id', 'ismap', 'label', 'lang', - 'longdesc', 'maxlength', 'media', 'method', 'multiple', 'name', - 'nohref', 'noshade', 'nowrap', 'prompt', 'readonly', 'rel', 'rev', - 'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size', - 'span', 'src', 'start', 'style', 'summary', 'tabindex', 'target', 'title', - 'type', 'usemap', 'valign', 'value', 'vspace', 'width', 'xml:lang'] - - - mathml_attributes = ['actiontype', 'align', 'columnalign', 'columnalign', - 'columnalign', 'columnlines', 'columnspacing', 'columnspan', 'depth', - 'display', 'displaystyle', 'equalcolumns', 'equalrows', 'fence', - 'fontstyle', 'fontweight', 'frame', 'height', 'linethickness', 'lspace', - 'mathbackground', 'mathcolor', 'mathvariant', 'mathvariant', 'maxsize', - 'minsize', 'other', 'rowalign', 'rowalign', 'rowalign', 'rowlines', - 'rowspacing', 'rowspan', 'rspace', 'scriptlevel', 'selection', - 'separator', 'stretchy', 'width', 'width', 'xlink:href', 'xlink:show', - 'xlink:type', 'xmlns', 'xmlns:xlink'] - - - svg_attributes = ['accent-height', 'accumulate', 'additive', 'alphabetic', - 'arabic-form', 'ascent', 'attributeName', 'attributeType', - 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', - 'class', 'color', 'color-rendering', 'content', 'cx', 'cy', 'd', 'dx', - 'dy', 'descent', 'display', 'dur', 'end', 'fill', 'fill-rule', - 'font-family', 'font-size', 'font-stretch', 'font-style', 'font-variant', - 'font-weight', 'from', 'fx', 'fy', 'g1', 'g2', 'glyph-name', - 'gradientUnits', 'hanging', 'height', 'horiz-adv-x', 'horiz-origin-x', - 'id', 'ideographic', 'k', 'keyPoints', 'keySplines', 'keyTimes', - 'lang', 'marker-end', 'marker-mid', 'marker-start', 'markerHeight', - 'markerUnits', 'markerWidth', 'mathematical', 'max', 'min', 'name', - 'offset', 'opacity', 'orient', 'origin', 'overline-position', - 'overline-thickness', 'panose-1', 'path', 'pathLength', 'points', - 'preserveAspectRatio', 'r', 'refX', 'refY', 'repeatCount', 'repeatDur', - 'requiredExtensions', 'requiredFeatures', 'restart', 'rotate', 'rx', - 'ry', 'slope', 'stemh', 'stemv', 'stop-color', 'stop-opacity', - 'strikethrough-position', 'strikethrough-thickness', 'stroke', - 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', - 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', - 'stroke-width', 'systemLanguage', 'target', - 'text-anchor', 'to', 'transform', 'type', 'u1', 'u2', - 'underline-position', 'underline-thickness', 'unicode', - 'unicode-range', 'units-per-em', 'values', 'version', 'viewBox', - 'visibility', 'width', 'widths', 'x', 'x-height', 'x1', 'x2', - 'xlink:actuate', 'xlink:arcrole', 'xlink:href', 'xlink:role', - 'xlink:show', 'xlink:title', 'xlink:type', 'xml:base', 'xml:lang', - 'xml:space', 'xmlns', 'xmlns:xlink', 'y', 'y1', 'y2', 'zoomAndPan'] - - attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'xlink:href'] - - acceptable_css_properties = ['azimuth', 'background-color', - 'border-bottom-color', 'border-collapse', 'border-color', - 'border-left-color', 'border-right-color', 'border-top-color', 'clear', - 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font', - 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', - 'height', 'letter-spacing', 'line-height', 'overflow', 'pause', - 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness', - 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', - 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent', - 'unicode-bidi', 'vertical-align', 'voice-family', 'volume', - 'white-space', 'width'] - - acceptable_css_keywords = ['auto', 'aqua', 'black', 'block', 'blue', - 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed', - 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', - 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive', - 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top', - 'transparent', 'underline', 'white', 'yellow'] - - acceptable_svg_properties = [ 'fill', 'fill-opacity', 'fill-rule', - 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', - 'stroke-opacity'] - - acceptable_protocols = [ 'ed2k', 'ftp', 'http', 'https', 'irc', - 'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal', - 'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag', - 'ssh', 'sftp', 'rtsp', 'afs' ] - - ALLOWED_ELEMENTS = acceptable_elements + mathml_elements + svg_elements unless defined?(ALLOWED_ELEMENTS) - ALLOWED_ATTRIBUTES = acceptable_attributes + mathml_attributes + svg_attributes unless defined?(ALLOWED_ATTRIBUTES) - ALLOWED_CSS_PROPERTIES = acceptable_css_properties unless defined?(ALLOWED_CSS_PROPERTIES) - ALLOWED_CSS_KEYWORDS = acceptable_css_keywords unless defined?(ALLOWED_CSS_KEYWORDS) - ALLOWED_SVG_PROPERTIES = acceptable_svg_properties unless defined?(ALLOWED_SVG_PROPERTIES) - ALLOWED_PROTOCOLS = acceptable_protocols unless defined?(ALLOWED_PROTOCOLS) - ATTR_VAL_IS_URI = attr_val_is_uri unless defined?(ATTR_VAL_IS_URI) - - # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and stripping out all - # attributes not in ALLOWED_ATTRIBUTES. Style attributes are parsed, and a restricted set, - # specified by ALLOWED_CSS_PROPERTIES and ALLOWED_CSS_KEYWORDS, are allowed through. - # attributes in ATTR_VAL_IS_URI are scanned, and only URI schemes specified in - # ALLOWED_PROTOCOLS are allowed. - # You can adjust what gets sanitized, by defining these constant arrays before this Module is loaded. - # - # sanitize_html('<script> do_nasty_stuff() </script>') - # => &lt;script> do_nasty_stuff() &lt;/script> - # sanitize_html('<a href="javascript: sucker();">Click here for $100</a>') - # => <a>Click here for $100</a> - def sanitize_html(html) - if html.index("<") - tokenizer = HTML::Tokenizer.new(html) - new_text = "" +# Uses the HTML5lib parser, so that the parsing behaviour should +# resemble that of browsers. +# +# sanitize_xhtml() is a case-sensitive sanitizer, suitable for XHTML +# sanitize_html() is a case-insensitive sanitizer suitable for HTML - while token = tokenizer.next - node = XHTML::Node.parse(nil, 0, 0, token, false) - new_text << case node.tag? - when true - if ALLOWED_ELEMENTS.include?(node.name) - if node.closing != :close - node.attributes.delete_if { |attr,v| !ALLOWED_ATTRIBUTES.include?(attr) } - ATTR_VAL_IS_URI.each do |attr| - val_unescaped = CGI.unescapeHTML(node.attributes[attr].to_s).gsub(/[\000-\040\177-\240]+/,'').downcase - if val_unescaped =~ /^[a-z0-9][-+.a-z0-9]*:/ and !ALLOWED_PROTOCOLS.include?(val_unescaped.split(':')[0]) - node.attributes.delete attr - end - end - if node.attributes['style'] - node.attributes['style'] = sanitize_css(node.attributes['style']) - end - end - node.to_s - else - node.to_s.gsub(/</, "&lt;") - end - else - node.to_s.gsub(/</, "&lt;") - end - end - html = new_text - end - html - end - - def sanitize_css(style) - # disallow urls - style = style.to_s.gsub(/url\s*\(\s*[^\s)]+?\s*\)\s*/, ' ') + require 'html5lib/sanitizer' + require 'html5lib/html5parser' + require 'html5lib/liberalxmlparser' + include HTML5lib - # gauntlet - if style !~ /^([:,;#%.\sa-zA-Z0-9!]|\w-\w|\'[\s\w]+\'|\"[\s\w]+\"|\([\d,\s]+\))*$/ - style = '' - return style - end - if style !~ /^(\s*[-\w]+\s*:\s*[^:;]*(;|$))*$/ - style = '' - return style - end + def sanitize_xhtml(html) + XHTMLParser.parseFragment(html, :tokenizer => HTMLSanitizer).to_s + end - clean = [] - style.scan(/([-\w]+)\s*:\s*([^:;]*)/) do |prop,val| - if ALLOWED_CSS_PROPERTIES.include?(prop.downcase) - clean << prop + ': ' + val + ';' - elsif ['background','border','margin','padding'].include?(prop.split('-')[0].downcase) - goodval = true - val.split().each do |keyword| - if !ALLOWED_CSS_KEYWORDS.include?(keyword) and - keyword !~ /^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$/ - goodval = false - end - end - if goodval - clean << prop + ': ' + val + ';' - end - elsif ALLOWED_SVG_PROPERTIES.include?(prop.downcase) - clean << prop + ': ' + val + ';' - end - end + def sanitize_html(html) + HTMLParser.parseFragment(html, :tokenizer => HTMLSanitizer).to_s + end - style = clean.join(' ') - end -end +end lib/sanitize.rb @@ -31,6 +31,9 @@ Globals = { :maruku_signature => false, :code_background_color => '#fef', :code_show_spaces => false, + + :filter_html => false, + :html_math_output_mathml => true, # also set :html_math_engine :html_math_engine => 'itex2mml', #ritex, itex2mml, none vendor/plugins/maruku/lib/maruku/defaults.rb @@ -477,7 +477,7 @@ module MaRuKu; module In; module Markdown; module BlockLevelParser end id = match[1]; url = match[2]; title = match[3]; - id = id.strip.downcase.gsub(' ','_') + id = sanitize_ref_id(id) hash = self.refs[id] = {:url=>url,:title=>title} vendor/plugins/maruku/lib/maruku/input/parse_block.rb @@ -287,7 +287,7 @@ module MaRuKu; module In; module Markdown; module SpanLevelParser end def extension_meta(src, con, break_on_chars) - if m = src.read_regexp(/([^\s\:]+):/) + if m = src.read_regexp(/([^\s\:\"\']+):/) name = m[1] al = read_attribute_list(src, con, break_on_chars) # puts "#{name}=#{al.inspect}" @@ -581,9 +581,9 @@ module MaRuKu; module In; module Markdown; module SpanLevelParser ref_id = read_ref_id(src,con) if ref_id if ref_id.size == 0 - ref_id = children.to_s.downcase.gsub(' ','_') + ref_id = sanitize_ref_id(children.to_s) else - ref_id = ref_id.downcase + ref_id = sanitize_ref_id(ref_id) end con.push_element md_link(children, ref_id) else vendor/plugins/maruku/lib/maruku/input/parse_span_better.rb @@ -108,6 +108,7 @@ module MaRuKu # Input is a LineSource def t2_parse_blocks(src, output) while src.cur_line + l = src.shift_line # ignore empty line if l.t2_empty? then @@ -115,7 +116,6 @@ module MaRuKu next end - l = src.shift_line # TODO: lists # TODO: xml # TODO: `==` vendor/plugins/maruku/lib/maruku/input_textile2/t2_parser.rb @@ -741,7 +741,17 @@ of the form `#ff00ff`. return a end +=begin maruku_doc +Attribute: filter_html +Scope: document + +If true, raw HTML is discarded from the output. + +=end + def to_html_raw_html + return [] if get_setting(:filter_html) + raw_html = self.raw_html if rexml_doc = @parsed_html root = rexml_doc.root vendor/plugins/maruku/lib/maruku/output/to_html.rb @@ -152,7 +152,7 @@ end end module MaRuKu; module Out; module Latex def to_latex_hrule; "\n\\vspace{.5em} \\hrule \\vspace{.5em}\n" end - def to_latex_linebreak; "\\linebreak " end + def to_latex_linebreak; "\\newline " end def to_latex_paragraph children_to_latex+"\n\n" vendor/plugins/maruku/lib/maruku/output/to_latex.rb @@ -146,6 +146,10 @@ module MaRuKu; module Strings s[0, i+1].strip end + def sanitize_ref_id(x) + x.downcase.gsub(' ','_').gsub(/[^\w]/,'') + end + # removes initial quote def unquote(s) vendor/plugins/maruku/lib/maruku/string_utils.rb @@ -155,7 +155,7 @@ module MaRuKu; module Tests ["[a]", [ md_link(["a"],'a')], 'Empty link'], ["[a][]", ], ["[a][]b", [ md_link(["a"],'a'),'b'], 'Empty link'], - ["[a\\]][]", [ md_link(["a]"],'a]')], 'Escape inside link'], + ["[a\\]][]", [ md_link(["a]"],'a')], 'Escape inside link (throw ?] away)'], ["[a", :throw, 'Link not closed'], ["[a][", :throw, 'Ref not closed'], vendor/plugins/maruku/lib/maruku/tests/new_parser.rb @@ -19,7 +19,7 @@ #++ module MaRuKu - Version = '0.5.5' + Version = '0.5.6' MarukuURL = 'http://maruku.rubyforge.org/' vendor/plugins/maruku/lib/maruku/version.rb lib/node.rb 457ec8627c3507e234a58bdb1fd5619d511e271b Jacques Distler distler@golem.ph.utexas.edu http://github.com/parasew/instiki/commit/6b21ac484f3e345723bc48d8e2a8d6753019b591 6b21ac484f3e345723bc48d8e2a8d6753019b591 2007-05-25T18:52:27-07:00 2007-05-25T18:52:27-07:00 HTML5lib Sanitizer Replaced native Sanitizer with HTML5lib version. Synced with latest Maruku. 76d48a469fd4fd00cfbf219c3957bc6a5c833dad Jacques Distler distler@golem.ph.utexas.edu