From 225255c703e462e2a43274b8afa14fa415d99bed Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Tue, 26 Nov 2013 17:46:46 -0800 Subject: [PATCH 1/4] Add initial proof-of-concept for HTML::Pipeline --- jekyll.gemspec | 4 ++ lib/jekyll/converters/markdown.rb | 4 +- .../markdown/html_pipeline_parser.rb | 58 +++++++++++++++++++ test/test_html_pipeline.rb | 42 ++++++++++++++ 4 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 lib/jekyll/converters/markdown/html_pipeline_parser.rb create mode 100644 test/test_html_pipeline.rb diff --git a/jekyll.gemspec b/jekyll.gemspec index 62a866a697a..bbcd2674af7 100644 --- a/jekyll.gemspec +++ b/jekyll.gemspec @@ -26,6 +26,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency('liquid', "~> 2.5.2") s.add_runtime_dependency('classifier', "~> 1.3") s.add_runtime_dependency('listen', "~> 1.3") + s.add_runtime_dependency('html-pipeline', "~> 1.0.0") s.add_runtime_dependency('maruku', "~> 0.6.0") s.add_runtime_dependency('pygments.rb', "~> 0.5.0") s.add_runtime_dependency('commander', "~> 4.1.3") @@ -41,8 +42,11 @@ Gem::Specification.new do |s| s.add_development_dependency('cucumber', "~> 1.3") s.add_development_dependency('RedCloth', "~> 4.2") s.add_development_dependency('kramdown', "~> 1.2") + s.add_development_dependency('github-markdown', "~> 0.6.3") s.add_development_dependency('rdiscount', "~> 1.6") s.add_development_dependency('launchy', "~> 2.3") + s.add_development_dependency('sanitize', "~> 2.0.6") + s.add_development_dependency('gemoji', "~> 1.5.0") s.add_development_dependency('simplecov', "~> 0.7") s.add_development_dependency('simplecov-gem-adapter', "~> 1.0.1") s.add_development_dependency('coveralls', "~> 0.7.0") diff --git a/lib/jekyll/converters/markdown.rb b/lib/jekyll/converters/markdown.rb index 485cac82374..f31efc94071 100644 --- a/lib/jekyll/converters/markdown.rb +++ b/lib/jekyll/converters/markdown.rb @@ -9,6 +9,8 @@ class Markdown < Converter def setup return if @setup @parser = case @config['markdown'] + when 'html_pipeline' + HTMLPipelineParser.new @config when 'redcarpet' RedcarpetParser.new @config when 'kramdown' @@ -19,7 +21,7 @@ def setup MarukuParser.new @config else STDERR.puts "Invalid Markdown processor: #{@config['markdown']}" - STDERR.puts " Valid options are [ maruku | rdiscount | kramdown | redcarpet ]" + STDERR.puts " Valid options are [ html_pipeline | maruku | rdiscount | kramdown | redcarpet ]" raise FatalException.new("Invalid Markdown process: #{@config['markdown']}") end @setup = true diff --git a/lib/jekyll/converters/markdown/html_pipeline_parser.rb b/lib/jekyll/converters/markdown/html_pipeline_parser.rb new file mode 100644 index 00000000000..62c5eef206b --- /dev/null +++ b/lib/jekyll/converters/markdown/html_pipeline_parser.rb @@ -0,0 +1,58 @@ +module Jekyll + module Converters + class Markdown + class HTMLPipelineParser + def initialize(config) + require 'html/pipeline' + @config = config + @errors = [] + end + + def filter_key(s) + s.to_s.downcase.to_sym + end + + def is_filter?(f) + f < HTML::Pipeline::Filter + rescue LoadError, ArgumentError + false + end + + def ensure_default_opts + @config['html_pipeline']['filters'] ||= ['markdownfilter'] + @config['html_pipeline']['context'] ||= {'gfm' => true} + # symbolize strings as keys, which is what HTML::Pipeline wants + @config['html_pipeline']['context'] = @config['html_pipeline']['context'].inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} + end + + def setup + unless @setup + ensure_default_opts + + filters = @config['html_pipeline']['filters'].map do |f| + if is_filter?(f) + f + else + key = filter_key(f) + begin + filter = HTML::Pipeline.constants.find { |c| c.downcase == key } + HTML::Pipeline.const_get(filter) + rescue Exception => e + raise FatalException.new(e) + end + end + end + + @parser = HTML::Pipeline.new(filters, @config['html_pipeline']['context']) + @setup = true + end + end + + def convert(content) + setup + @parser.to_html(content) + end + end + end + end +end diff --git a/test/test_html_pipeline.rb b/test/test_html_pipeline.rb new file mode 100644 index 00000000000..b74731e0413 --- /dev/null +++ b/test/test_html_pipeline.rb @@ -0,0 +1,42 @@ +# encoding: UTF-8 + +require 'helper' + +class TestHTMLPipeline < Test::Unit::TestCase + context "HTMLPipeline" do + setup do + @config = { + 'markdown' => 'html_pipeline', + 'html_pipeline' => { + 'filters' => ['markdownfilter', 'sanitizationfilter', 'emojifilter', 'mentionfilter'], + 'context' => { + 'asset_root' => "http://foo.com/icons" + } + } + } + + @config = Jekyll.configuration(@config) + @markdown = Converters::Markdown.new(@config) + end + + should "pass regular options" do + assert_equal "

Some Header

", @markdown.convert('# Some Header #').strip + end + + should "pass render emoji" do + assert_equal "

\":trollface:\"

", @markdown.convert(':trollface:').strip + end + + should "fail when a library dependency is not met" do + override = { 'html_pipeline' => { 'filters' => ['markdownfilter, AutolinkFilter'] } } + markdown = Converters::Markdown.new(@config.deep_merge(override)) + # assert_fail markdown.convert('http://www.github.com') + end + + should "fail when a context dependency is not met" do + override = { 'html_pipeline' => { 'context' => {} } } + markdown = Converters::Markdown.new(@config.deep_merge(override)) + # assert_fail markdown.convert(':trollface:') + end + end +end From a012a63d3e18c2848e00719461ab857e956c887a Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Tue, 26 Nov 2013 18:41:55 -0800 Subject: [PATCH 2/4] Add test for mentions --- test/test_html_pipeline.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test_html_pipeline.rb b/test/test_html_pipeline.rb index b74731e0413..40463287c5c 100644 --- a/test/test_html_pipeline.rb +++ b/test/test_html_pipeline.rb @@ -10,7 +10,8 @@ class TestHTMLPipeline < Test::Unit::TestCase 'html_pipeline' => { 'filters' => ['markdownfilter', 'sanitizationfilter', 'emojifilter', 'mentionfilter'], 'context' => { - 'asset_root' => "http://foo.com/icons" + 'asset_root' => "http://foo.com/icons", + 'base_url' => "https://github.com/", } } } @@ -23,10 +24,14 @@ class TestHTMLPipeline < Test::Unit::TestCase assert_equal "

Some Header

", @markdown.convert('# Some Header #').strip end - should "pass render emoji" do + should "pass rendering emoji" do assert_equal "

\":trollface:\"

", @markdown.convert(':trollface:').strip end + should "pass rendering mentions" do + assert_equal "

Hey, @mojombo!

", @markdown.convert('**Hey, @mojombo**!').strip + end + should "fail when a library dependency is not met" do override = { 'html_pipeline' => { 'filters' => ['markdownfilter, AutolinkFilter'] } } markdown = Converters::Markdown.new(@config.deep_merge(override)) From 9db3d50f5d5cb28f3cf6aa729fe7620411bfda5f Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Tue, 26 Nov 2013 19:09:22 -0800 Subject: [PATCH 3/4] Address comments --- lib/jekyll/converters/markdown.rb | 2 +- lib/jekyll/converters/markdown/html_pipeline_parser.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jekyll/converters/markdown.rb b/lib/jekyll/converters/markdown.rb index f31efc94071..4e96800df31 100644 --- a/lib/jekyll/converters/markdown.rb +++ b/lib/jekyll/converters/markdown.rb @@ -21,7 +21,7 @@ def setup MarukuParser.new @config else STDERR.puts "Invalid Markdown processor: #{@config['markdown']}" - STDERR.puts " Valid options are [ html_pipeline | maruku | rdiscount | kramdown | redcarpet ]" + STDERR.puts " Valid options are [ maruku | rdiscount | kramdown | redcarpet | html_pipeline ]" raise FatalException.new("Invalid Markdown process: #{@config['markdown']}") end @setup = true diff --git a/lib/jekyll/converters/markdown/html_pipeline_parser.rb b/lib/jekyll/converters/markdown/html_pipeline_parser.rb index 62c5eef206b..09aa59c2817 100644 --- a/lib/jekyll/converters/markdown/html_pipeline_parser.rb +++ b/lib/jekyll/converters/markdown/html_pipeline_parser.rb @@ -22,7 +22,7 @@ def ensure_default_opts @config['html_pipeline']['filters'] ||= ['markdownfilter'] @config['html_pipeline']['context'] ||= {'gfm' => true} # symbolize strings as keys, which is what HTML::Pipeline wants - @config['html_pipeline']['context'] = @config['html_pipeline']['context'].inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo} + @config['html_pipeline']['context'] = @config['html_pipeline']['context'].symbolize_keys end def setup From 85939fdbb54ea1de6b5c7ca4c54cfe93b0edac0e Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Tue, 26 Nov 2013 22:39:30 -0800 Subject: [PATCH 4/4] Move html-pipeline to a dev dependency --- jekyll.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jekyll.gemspec b/jekyll.gemspec index bbcd2674af7..b52ac5b6109 100644 --- a/jekyll.gemspec +++ b/jekyll.gemspec @@ -26,7 +26,6 @@ Gem::Specification.new do |s| s.add_runtime_dependency('liquid', "~> 2.5.2") s.add_runtime_dependency('classifier', "~> 1.3") s.add_runtime_dependency('listen', "~> 1.3") - s.add_runtime_dependency('html-pipeline', "~> 1.0.0") s.add_runtime_dependency('maruku', "~> 0.6.0") s.add_runtime_dependency('pygments.rb', "~> 0.5.0") s.add_runtime_dependency('commander', "~> 4.1.3") @@ -42,6 +41,7 @@ Gem::Specification.new do |s| s.add_development_dependency('cucumber', "~> 1.3") s.add_development_dependency('RedCloth', "~> 4.2") s.add_development_dependency('kramdown', "~> 1.2") + s.add_development_dependency('html-pipeline', "~> 1.0.0") s.add_development_dependency('github-markdown', "~> 0.6.3") s.add_development_dependency('rdiscount', "~> 1.6") s.add_development_dependency('launchy', "~> 2.3")