Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

add really basic v0.1 of Pygments::Popen

  • Loading branch information...
commit 712644d7d2d03ac946eecbc3262fd7fb246931f4 1 parent 2e55087
@tmm1 tmm1 authored
View
10 lib/pygments.rb
@@ -1,11 +1,11 @@
# require 'pygments/c'
-require 'pygments/ffi'
+# require 'pygments/ffi'
+require 'pygments/popen'
module Pygments
- # include Pygments::C
- include Pygments::FFI
+ # extend Pygments::C
+ # extend Pygments::FFI
+ extend Pygments::Popen
autoload :Lexer, 'pygments/lexer'
-
- extend self
end
View
90 lib/pygments/popen.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+
+import sys, os
+if 'PYGMENTS_PATH' in os.environ:
+ sys.path.insert(0, os.environ['PYGMENTS_PATH'])
+
+import pygments
+import pygments.lexers, pygments.formatters, pygments.styles, pygments.filters
+
+import re
+import json
+
+def lexer_for(code=None, **opts):
+ lexers = pygments.lexers
+ kwargs = opts.get('options') or dict()
+
+ if 'lexer' in opts:
+ return lexers.get_lexer_by_name(opts['lexer'], **kwargs)
+ elif 'mimetype' in opts:
+ return lexers.get_lexer_for_mimetype(opts['mimetype'], **kwargs)
+ elif 'filename' in opts:
+ name = opts['filename']
+ if code:
+ return lexers.guess_lexer_for_filename(name, code, **kwargs)
+ else:
+ return lexers.get_lexer_for_filename(name, **kwargs)
+ elif code:
+ return lexers.guess_lexer(code, **kwargs)
+ else:
+ return None
+
+while True:
+ line = sys.stdin.readline()
+
+ req = json.loads(line)
+ res = None
+
+ if 'method' in req:
+ method = req['method']
+
+ if method == 'get_all_styles':
+ res = [ s for s in pygments.styles.get_all_styles() ]
+
+ elif method == 'get_all_filters':
+ res = [ f for f in pygments.filters.get_all_filters() ]
+
+ elif method == 'get_all_lexers':
+ res = [ l for l in pygments.lexers.get_all_lexers() ]
+
+ elif method == 'get_all_formatters':
+ res = [ [f.__name__, f.name, f.aliases] for f in pygments.formatters.get_all_formatters() ]
+
+ elif method == 'highlight':
+ code = req['args']
+ opts = req['kwargs']
+ kwargs = opts.get('options') or dict()
+
+ lexer = lexer_for(code, **opts)
+
+ fmt_name = str.lower(str(opts.get('formatter') or 'html'))
+ formatter = pygments.formatters.get_formatter_by_name(fmt_name, **kwargs)
+
+ res = pygments.highlight(code, lexer, formatter)
+
+ if res and fmt_name == 'html':
+ res = re.sub(r'</pre></div>\n?\Z', "</pre>\n</div>", res)
+
+ elif method == 'css':
+ args = req['args']
+ kwargs = req.get('kwargs') or dict()
+
+ fmt = pygments.formatters.get_formatter_by_name(args[0], **kwargs)
+ res = fmt.get_style_defs(args[1])
+
+ elif method == 'lexer_name_for':
+ args = req['args']
+ kwargs = req.get('kwargs') or dict()
+
+ lxr = lexer_for(*args, **kwargs)
+ if lxr:
+ res = lxr.aliases[0]
+
+ else:
+ res = dict(error="invalid method")
+
+ if res:
+ sys.stdout.write(json.dumps(res))
+ sys.stdout.write("\n")
+ sys.stdout.flush()
+
View
115 lib/pygments/popen.rb
@@ -0,0 +1,115 @@
+require 'posix/spawn'
+require 'yajl'
+
+module Pygments
+ module Popen
+ include POSIX::Spawn
+ extend self
+
+ def start(pygments_path = File.expand_path('../../../vendor/pygments-main/', __FILE__))
+ ENV['PYGMENTS_PATH'] = pygments_path
+ at_exit{ stop }
+ @pid, @in, @out, @err = popen4(File.expand_path('../popen.py', __FILE__))
+ end
+
+ def stop
+ if @pid
+ begin
+ Process.kill 'KILL', @pid
+ Process.waitpid @pid
+ rescue Errno::ESRCH
+ end
+ end
+
+ @pid = nil
+ end
+
+ def alive?
+ return true if @pid && Process.kill(0, @pid)
+ false
+ rescue Errno::ENOENT, Errno::ESRCH
+ false
+ end
+
+ def formatters
+ rpc(:get_all_formatters).inject(Hash.new) do |hash, (name, desc, aliases)|
+ name.sub!(/Formatter$/,'')
+ hash[name] = {
+ :name => name,
+ :description => desc,
+ :aliases => aliases
+ }
+ hash
+ end
+ end
+
+ def lexers
+ rpc(:get_all_lexers).inject(Hash.new) do |hash, lxr|
+ name = lxr[0]
+ hash[name] = {
+ :name => name,
+ :aliases => lxr[1],
+ :filenames => lxr[2],
+ :mimetypes => lxr[3]
+ }
+ hash
+ end
+ end
+
+ def filters
+ rpc(:get_all_filters)
+ end
+
+ def styles
+ rpc(:get_all_styles)
+ end
+
+ def css(klass='', opts={})
+ if klass.is_a?(Hash)
+ opts = klass
+ klass = ''
+ end
+ rpc(:css, ['html', klass], opts)
+ end
+
+ def lexer_name_for(*args)
+ opts = args.pop if args.last.is_a?(Hash)
+ rpc(:lexer_name_for, args, opts)
+ end
+
+ def highlight(code, opts={})
+ if code.nil? or code.empty?
+ return code
+ end
+
+ opts[:options] ||= {}
+ opts[:options][:outencoding] ||= 'utf-8'
+
+ str = rpc(:highlight, code, opts)
+ str.force_encoding(opts[:options][:outencoding]) if str.respond_to?(:force_encoding)
+ str
+ end
+
+ private
+
+ def rpc(method, args=nil, kwargs=nil)
+ start unless alive?
+
+ req = Yajl.dump(:method => method, :args => args, :kwargs => kwargs)
+ # use select/read+write loop (copy posix-spawn)
+ @in.puts(req)
+ if res = @out.gets and res.any?
+ res = Yajl.load(res, :symbolize_keys => true)
+ else
+ p @err.read
+ res = nil
+ end
+
+ res
+ rescue Errno::EPIPE, EOFError
+ p $!
+ # retry
+ raise
+ end
+ end
+end
View
6 pygments.rb.gemspec
@@ -13,8 +13,10 @@ Gem::Specification.new do |s|
s.authors = ['Aman Gupta']
s.email = ['aman@tmm1.net']
- s.add_dependency 'rubypython', '~> 0.5.3'
- s.add_development_dependency 'rake-compiler', '0.7.6'
+ s.add_dependency 'rubypython', '~> 0.5.3'
+ s.add_dependency 'yajl-ruby', '~> 1.1.0'
+ s.add_dependency 'posix-spawn', '~> 0.3.6'
+ s.add_development_dependency 'rake-compiler', '~> 0.7.6'
# s.extensions = ['ext/extconf.rb']
s.require_paths = ['lib']
View
88 test/test_pygments.rb
@@ -3,57 +3,55 @@
require 'test/unit'
require 'pygments'
-class PygmentsLexerTest < Test::Unit::TestCase
- include Pygments
+P = Pygments
+class PygmentsLexerTest < Test::Unit::TestCase
RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"
def test_lexer_by_content
- assert_equal 'rb', lexer_name_for(RUBY_CODE)
+ assert_equal 'rb', P.lexer_name_for(RUBY_CODE)
end
def test_lexer_by_mimetype
- assert_equal 'rb', lexer_name_for(:mimetype => 'text/x-ruby')
+ assert_equal 'rb', P.lexer_name_for(:mimetype => 'text/x-ruby')
end
def test_lexer_by_filename
- assert_equal 'rb', lexer_name_for(:filename => 'test.rb')
+ assert_equal 'rb', P.lexer_name_for(:filename => 'test.rb')
end
def test_lexer_by_name
- assert_equal 'rb', lexer_name_for(:lexer => 'ruby')
+ assert_equal 'rb', P.lexer_name_for(:lexer => 'ruby')
end
def test_lexer_by_filename_and_content
- assert_equal 'rb', lexer_name_for(RUBY_CODE, :filename => 'test.rb')
+ assert_equal 'rb', P.lexer_name_for(RUBY_CODE, :filename => 'test.rb')
end
def test_lexer_by_nothing
- assert_equal nil, lexer_name_for(:invalid => true)
+ assert_equal nil, P.lexer_name_for(:invalid => true)
end
end
class PygmentsLexerClassTest < Test::Unit::TestCase
- include Pygments
-
def test_find
- assert_equal 'Ruby', Lexer['Ruby'].name
- assert_equal 'Ruby', Lexer['ruby'].name
- assert_equal 'Ruby', Lexer['rb'].name
- assert_equal 'Ruby', Lexer['rake'].name
- assert_equal 'Ruby', Lexer['gemspec'].name
+ assert_equal 'Ruby', P::Lexer['Ruby'].name
+ assert_equal 'Ruby', P::Lexer['ruby'].name
+ assert_equal 'Ruby', P::Lexer['rb'].name
+ assert_equal 'Ruby', P::Lexer['rake'].name
+ assert_equal 'Ruby', P::Lexer['gemspec'].name
end
def test_find_by_name
- assert_equal Lexer['Ruby'], Lexer.find_by_name('Ruby')
+ assert_equal P::Lexer['Ruby'], P::Lexer.find_by_name('Ruby')
end
def test_find_by_alias
- assert_equal Lexer['Ruby'], Lexer.find_by_alias('rb')
- assert_equal Lexer['Ruby'], Lexer.find_by_alias('ruby')
+ assert_equal P::Lexer['Ruby'], P::Lexer.find_by_alias('rb')
+ assert_equal P::Lexer['Ruby'], P::Lexer.find_by_alias('ruby')
end
def test_find_lexer_by_extname
- assert_equal Lexer['Ruby'], Lexer.find_by_extname('.rb')
- assert_equal Lexer['PHP'], Lexer.find_by_extname('.php4')
- assert_equal Lexer['PHP'], Lexer.find_by_extname('.php5')
- assert_equal Lexer['Groff'], Lexer.find_by_extname('.1')
- assert_equal Lexer['Groff'], Lexer.find_by_extname('.3')
+ assert_equal P::Lexer['Ruby'], P::Lexer.find_by_extname('.rb')
+ assert_equal P::Lexer['PHP'], P::Lexer.find_by_extname('.php4')
+ assert_equal P::Lexer['PHP'], P::Lexer.find_by_extname('.php5')
+ assert_equal P::Lexer['Groff'], P::Lexer.find_by_extname('.1')
+ assert_equal P::Lexer['Groff'], P::Lexer.find_by_extname('.3')
end
def test_find_lexer_by_mimetype
- assert_equal Lexer['Ruby'], Lexer.find_by_mimetype('text/x-ruby')
+ assert_equal P::Lexer['Ruby'], P::Lexer.find_by_mimetype('text/x-ruby')
end
end
@@ -61,88 +59,84 @@ class PygmentsCssTest < Test::Unit::TestCase
include Pygments
def test_css
- assert_match /^\.err \{/, css
+ assert_match /^\.err \{/, P.css
end
def test_css_prefix
- assert_match /^\.highlight \.err \{/, css('.highlight')
+ assert_match /^\.highlight \.err \{/, P.css('.highlight')
end
def test_css_options
- assert_match /^\.codeerr \{/, css(:classprefix => 'code')
+ assert_match /^\.codeerr \{/, P.css(:classprefix => 'code')
end
def test_css_prefix_and_options
- assert_match /^\.mycode \.codeerr \{/, css('.mycode', :classprefix => 'code')
+ assert_match /^\.mycode \.codeerr \{/, P.css('.mycode', :classprefix => 'code')
end
def test_css_default
- assert_match '.c { color: #408080; font-style: italic }', css
+ assert_match '.c { color: #408080; font-style: italic }', P.css
end
def test_css_colorful
- assert_match '.c { color: #808080 }', css(:style => 'colorful')
+ assert_match '.c { color: #808080 }', P.css(:style => 'colorful')
end
end
class PygmentsConfigTest < Test::Unit::TestCase
- include Pygments
-
def test_styles
- assert styles.include?('colorful')
+ assert P.styles.include?('colorful')
end
def test_filters
- assert filters.include?('codetagify')
+ assert P.filters.include?('codetagify')
end
def test_lexers
- list = lexers
+ list = P.lexers
assert list.has_key?('Ruby')
assert list['Ruby'][:aliases].include?('duby')
end
def test_formatters
- list = formatters
+ list = P.formatters
assert list.has_key?('Html')
assert list['Html'][:aliases].include?('html')
end
end
class PygmentsHighlightTest < Test::Unit::TestCase
- include Pygments
-
RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"
def test_highlight_empty
- highlight('')
- highlight(nil)
+ P.highlight('')
+ P.highlight(nil)
end
def test_highlight_defaults_to_html
- code = highlight(RUBY_CODE)
+ code = P.highlight(RUBY_CODE)
assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
end
def test_highlight_markdown_compatible_html
- code = highlight(RUBY_CODE)
+ code = P.highlight(RUBY_CODE)
assert_no_match %r{</pre></div>\Z}, code
end
def test_highlight_works_with_null_bytes
- code = highlight("\0hello", :lexer => 'rb')
+ code = P.highlight("\0hello", :lexer => 'rb')
assert_match "hello", code
end
def test_highlight_works_on_utf8
- code = highlight('# ø', :lexer => 'rb', :options => {:encoding => 'utf-8'})
+ code = P.highlight('# ø', :lexer => 'rb', :options => {:encoding => 'utf-8'})
assert_match '<span class="c1"># ø</span>', code
end
def test_highlight_formatter_bbcode
- code = highlight(RUBY_CODE, :formatter => 'bbcode')
+ code = P.highlight(RUBY_CODE, :formatter => 'bbcode')
assert_match '[i]#!/usr/bin/ruby[/i]', code
end
def test_highlight_formatter_terminal
- code = highlight(RUBY_CODE, :formatter => 'terminal')
+ code = P.highlight(RUBY_CODE, :formatter => 'terminal')
assert_match "\e[37m#!/usr/bin/ruby\e[39;49;00m", code
end
def test_highlight_options
- code = highlight(RUBY_CODE, :options => {:full => true, :title => 'test'})
+ code = P.highlight(RUBY_CODE, :options => {:full => true, :title => 'test'})
assert_match '<title>test</title>', code
end
end
Please sign in to comment.
Something went wrong with that request. Please try again.