add really basic v0.1 of Pygments::Popen

lib/pygments.rb

@@ -1,11 +1,11 @@
 # require 'pygments/c'
-require 'pygments/ffi'
+# require 'pygments/ffi'
+require 'pygments/popen'
 
 module Pygments
-  # include Pygments::C
-  include Pygments::FFI
+  # extend Pygments::C
+  # extend Pygments::FFI
+  extend Pygments::Popen
 
   autoload :Lexer, 'pygments/lexer'
-
-  extend self
 end

lib/pygments/popen.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+
+import sys, os
+if 'PYGMENTS_PATH' in os.environ:
+  sys.path.insert(0, os.environ['PYGMENTS_PATH'])
+
+import pygments
+import pygments.lexers, pygments.formatters, pygments.styles, pygments.filters
+
+import re
+import json
+
+def lexer_for(code=None, **opts):
+  lexers = pygments.lexers
+  kwargs = opts.get('options') or dict()
+
+  if 'lexer' in opts:
+    return lexers.get_lexer_by_name(opts['lexer'], **kwargs)
+  elif 'mimetype' in opts:
+    return lexers.get_lexer_for_mimetype(opts['mimetype'], **kwargs)
+  elif 'filename' in opts:
+    name = opts['filename']
+    if code:
+      return lexers.guess_lexer_for_filename(name, code, **kwargs)
+    else:
+      return lexers.get_lexer_for_filename(name, **kwargs)
+  elif code:
+    return lexers.guess_lexer(code, **kwargs)
+  else:
+    return None
+
+while True:
+  line = sys.stdin.readline()
+
+  req = json.loads(line)
+  res = None
+
+  if 'method' in req:
+    method = req['method']
+
+    if   method == 'get_all_styles':
+      res = [ s for s in pygments.styles.get_all_styles() ]
+
+    elif method == 'get_all_filters':
+      res = [ f for f in pygments.filters.get_all_filters() ]
+
+    elif method == 'get_all_lexers':
+      res = [ l for l in pygments.lexers.get_all_lexers() ]
+
+    elif method == 'get_all_formatters':
+      res = [ [f.__name__, f.name, f.aliases] for f in pygments.formatters.get_all_formatters() ]
+
+    elif method == 'highlight':
+      code = req['args']
+      opts = req['kwargs']
+      kwargs = opts.get('options') or dict()
+
+      lexer = lexer_for(code, **opts)
+
+      fmt_name = str.lower(str(opts.get('formatter') or 'html'))
+      formatter = pygments.formatters.get_formatter_by_name(fmt_name, **kwargs)
+
+      res = pygments.highlight(code, lexer, formatter)
+
+      if res and fmt_name == 'html':
+        res = re.sub(r'</pre></div>\n?\Z', "</pre>\n</div>", res)
+
+    elif method == 'css':
+      args = req['args']
+      kwargs = req.get('kwargs') or dict()
+
+      fmt = pygments.formatters.get_formatter_by_name(args[0], **kwargs)
+      res = fmt.get_style_defs(args[1])
+
+    elif method == 'lexer_name_for':
+      args = req['args']
+      kwargs = req.get('kwargs') or dict()
+
+      lxr = lexer_for(*args, **kwargs)
+      if lxr:
+        res = lxr.aliases[0]
+
+    else:
+      res = dict(error="invalid method")
+
+  if res:
+    sys.stdout.write(json.dumps(res))
+  sys.stdout.write("\n")
+  sys.stdout.flush()
+

lib/pygments/popen.rb

@@ -0,0 +1,115 @@
+require 'posix/spawn'
+require 'yajl'
+
+module Pygments
+  module Popen
+    include POSIX::Spawn
+    extend self
+
+    def start(pygments_path = File.expand_path('../../../vendor/pygments-main/', __FILE__))
+      ENV['PYGMENTS_PATH'] = pygments_path
+      at_exit{ stop }
+      @pid, @in, @out, @err = popen4(File.expand_path('../popen.py', __FILE__))
+    end
+
+    def stop
+      if @pid
+        begin
+          Process.kill 'KILL', @pid
+          Process.waitpid @pid
+        rescue Errno::ESRCH
+        end
+      end
+
+      @pid = nil
+    end
+
+    def alive?
+      return true if @pid && Process.kill(0, @pid)
+      false
+    rescue Errno::ENOENT, Errno::ESRCH
+      false
+    end
+
+    def formatters
+      rpc(:get_all_formatters).inject(Hash.new) do |hash, (name, desc, aliases)|
+        name.sub!(/Formatter$/,'')
+        hash[name] = {
+          :name => name,
+          :description => desc,
+          :aliases => aliases
+        }
+        hash
+      end
+    end
+
+    def lexers
+      rpc(:get_all_lexers).inject(Hash.new) do |hash, lxr|
+        name = lxr[0]
+        hash[name] = {
+          :name => name,
+          :aliases => lxr[1],
+          :filenames => lxr[2],
+          :mimetypes => lxr[3]
+        }
+        hash
+      end
+    end
+
+    def filters
+      rpc(:get_all_filters)
+    end
+
+    def styles
+      rpc(:get_all_styles)
+    end
+
+    def css(klass='', opts={})
+      if klass.is_a?(Hash)
+        opts = klass
+        klass = ''
+      end
+      rpc(:css, ['html', klass], opts)
+    end
+
+    def lexer_name_for(*args)
+      opts = args.pop if args.last.is_a?(Hash)
+      rpc(:lexer_name_for, args, opts)
+    end
+
+    def highlight(code, opts={})
+      if code.nil? or code.empty?
+        return code
+      end
+
+      opts[:options] ||= {}
+      opts[:options][:outencoding] ||= 'utf-8'
+
+      str = rpc(:highlight, code, opts)
+      str.force_encoding(opts[:options][:outencoding]) if str.respond_to?(:force_encoding)
+      str
+    end
+
+    private
+
+    def rpc(method, args=nil, kwargs=nil)
+      start unless alive?
+
+      req = Yajl.dump(:method => method, :args => args, :kwargs => kwargs)
+      # use select/read+write loop (copy posix-spawn)
+      @in.puts(req)
+      if res = @out.gets and res.any?
+        res = Yajl.load(res, :symbolize_keys => true)
+      else
+        p @err.read
+        res = nil
+      end
+
+      res
+    rescue Errno::EPIPE, EOFError
+      p $!
+      # retry
+      raise
+    end
+  end
+end

pygments.rb.gemspec

@@ -13,8 +13,10 @@ Gem::Specification.new do |s|
   s.authors = ['Aman Gupta']
   s.email = ['aman@tmm1.net']
 
-  s.add_dependency 'rubypython', '~> 0.5.3'
-  s.add_development_dependency 'rake-compiler', '0.7.6'
+  s.add_dependency 'rubypython',  '~> 0.5.3'
+  s.add_dependency 'yajl-ruby',   '~> 1.1.0'
+  s.add_dependency 'posix-spawn', '~> 0.3.6'
+  s.add_development_dependency 'rake-compiler', '~> 0.7.6'
 
   # s.extensions = ['ext/extconf.rb']
   s.require_paths = ['lib']