Version 0.3, using pipes for IPC

.gitignore

@@ -3,3 +3,4 @@ ext/Makefile
 lib/pygments_ext.*
 tmp
 pkg
+*.pyc

README.md

@@ -1,14 +1,23 @@
 # pygments.rb
 
-A ruby wrapper for the python [pygments syntax highlighter](http://pygments.org/).
+A Ruby wrapper for the Python [pygments syntax highlighter](http://pygments.org/).
 
-This library replaces [github/albino](https://github.com/github/albino).
-Instead of shelling out to `pygmentize`, it embeds the python
-interpreter inside ruby via FFI. This avoids the cost of setting up the
-python VM on every invocation and speeds up code highlighting from ruby by 10-15x.
+pygments.rb works by talking over a simple pipe to a long-lived 
+Python child process. This library replaces [github/albino](https://github.com/github/albino),
+as well as a version of pygments.rb that used an embedded Python
+interpreter. 
+
+Each Ruby process that runs has its own 'personal Python'; 
+for example, 4 Unicorn workers will have one Python process each. 
+If a Python process dies, a new one will be spawned on the next 
+pygments.rb request. 
 
 ## usage
 
+``` ruby 
+require 'pygments'
+``` 
+
 ``` ruby
 Pygments.highlight(File.read(__FILE__), :lexer => 'ruby')
 ```
@@ -20,42 +29,59 @@ options hash:
 Pygments.highlight('code', :options => {:encoding => 'utf-8'})
 ```
 
-To use a formatter other than html, specify it explicitly:
+pygments.rb defaults to using an HTML formatter. 
+To use a formatter other than `html`, specify it explicitly
+like so:
 
 ``` ruby
 Pygments.highlight('code', :formatter => 'bbcode')
 Pygments.highlight('code', :formatter => 'terminal')
 ```
 
-To generate CSS for html formatted code, use the css method:
+To generate CSS for HTML formatted code, use the `#css` method:
 
 ``` ruby
 Pygments.css
 Pygments.css('.highlight')
 ```
 
-To use a custom python installation (like in ArchLinux), tell
-RubyPython where python lives:
+Other Pygments high-level API methods are also available.
+These methods return arrays detailing all the available lexers, formatters, 
+and styles.
 
 ``` ruby
-RubyPython.configure :python_exe => 'python2.7'
+Pygments.lexers
+Pygments.formatters
+Pygments.styles
 ```
 
 To use a custom pygments installation, specify the path to
-Pygments.start:
+`Pygments#start`:
 
 ``` ruby
 Pygments.start("/path/to/pygments")
 ```
 
 ## benchmarks
 
-    $ ruby -rubygems bench.rb 50
-                                   user     system      total        real
-    albino                     0.050000   0.050000  12.830000 ( 13.180806)
-    pygments::c                1.000000   0.010000   1.010000 (  1.009348)
-    pygments::ffi + reload    11.350000   1.240000  12.590000 ( 12.692320)
-    pygments::ffi              1.130000   0.010000   1.140000 (  1.171589)
 
-To run `bench.rb`, use a git checkout. The C extension is not included
-in gem releases.
+    $ ruby bench.rb 50
+       Benchmarking....
+       Size: 698 bytes
+       Iterations: 50
+                                                      user     system      total        real
+       pygments popen                                0.010000   0.010000   0.020000 (  0.460370)
+       pygments popen (process already started)      0.010000   0.000000   0.010000 (  0.272975)
+       pygments popen (process already started 2)    0.000000   0.000000   0.000000 (  0.273589)
+
+    $ ruby bench.rb 10
+       Benchmarking....
+       Size: 15523 bytes
+       Iterations: 10
+                                                      user     system      total        real
+       pygments popen                               0.000000   0.000000   0.000000 (  0.819419)
+       pygments popen (process already started)     0.010000   0.000000   0.010000 (  0.676515)
+       pygments popen (process already started 2)   0.000000   0.010000   0.010000 (  0.674189)
+
+
+

Rakefile

@@ -1,3 +1,6 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
 task :default => :test
 
 # ==========================================================
@@ -10,16 +13,6 @@ require 'rake/gempackagetask'
 Rake::GemPackageTask.new(GEMSPEC) do |pkg|
 end
 
-# ==========================================================
-# Ruby Extension
-# ==========================================================
-
-require 'rake/extensiontask'
-Rake::ExtensionTask.new('pygments_ext', GEMSPEC) do |ext|
-  ext.ext_dir = 'ext'
-end
-task :build => :compile
-
 # ==========================================================
 # Testing
 # ==========================================================
@@ -29,7 +22,24 @@ Rake::TestTask.new 'test' do |t|
   t.test_files = FileList['test/test_*.rb']
   t.ruby_opts = ['-rubygems']
 end
-task :test => :build
+
+# ==========================================================
+# Benchmarking
+# ==========================================================
+
+task :bench do
+  sh "ruby bench.rb"
+end
+
+# ==========================================================
+# Cache lexers
+# # ==========================================================
+
+# Write all the lexers to a file for easy lookup
+task :lexers do
+  sh "ruby cache-lexers.rb"
+end
+
 
 # ==========================================================
 # Vendor

bench.rb

@@ -1,55 +1,22 @@
-$:.unshift('lib')
-
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
 require 'benchmark'
-require 'pygments/c'
-require 'pygments/ffi'
-require 'rubygems'
-require 'albino'
 
-num = ARGV[0] ? ARGV[0].to_i : 25
-code = File.read(__FILE__)
+include Benchmark
+# number of iterations
+num = ARGV[0] ? ARGV[0].to_i : 10
 
-albino, pygments, ffi =
-  Albino.new(code, :ruby, :html).colorize,
-  Pygments::C.highlight(code, :lexer => 'ruby'),
-  Pygments::FFI.highlight(code, :lexer => 'ruby')
+# we can also repeat the code itself
+repeats = ARGV[1] ? ARGV[1].to_i : 1
 
-unless albino == pygments and pygments == ffi
-  raise "incompatible implementations (#{albino.size} != #{pygments.size} != #{ffi.size})"
-end
+code = File.open('test/test_data.py').read.to_s * repeats
 
-Benchmark.bm(25) do |x|
-  x.report('albino') do
-    num.times do
-      Albino.new(code, :ruby, :html).colorize
-    end
-  end
-  x.report('pygments::c') do
-    num.times do
-      Pygments::C.highlight(code, :lexer => 'ruby')
-    end
-  end
-  x.report('pygments::ffi + reload') do
-    num.times do
-      Pygments::FFI.start
-      Pygments::FFI.highlight(code, :lexer => 'ruby')
-      Pygments::FFI.stop
-    end
-  end
-  Pygments::FFI.start
-  x.report('pygments::ffi') do
-    num.times do
-      Pygments::FFI.highlight(code, :lexer => 'ruby')
-    end
-  end
-end
+puts "Benchmarking....\n"
+puts "Size: " + code.bytesize.to_s + " bytes\n"
+puts "Iterations: " + num.to_s + "\n"
 
-__END__
-
-$ ruby -rubygems bench.rb 50
-                               user     system      total        real
-albino                     0.050000   0.050000  12.830000 ( 13.180806)
-pygments::c                1.000000   0.010000   1.010000 (  1.009348)
-pygments::ffi + reload    11.350000   1.240000  12.590000 ( 12.692320)
-pygments::ffi              1.130000   0.010000   1.140000 (  1.171589)
+Benchmark.bm(40) do |x|
+  x.report("pygments popen                             ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+  x.report("pygments popen (process already started)   ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+  x.report("pygments popen (process already started 2) ")  { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+end
 

cache-lexers.rb

@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
+
+# Simple marshalling
+serialized_lexers = Marshal.dump(Pygments.lexers!)
+
+# Write to a file
+File.open("lexers", 'w') { |file| file.write(serialized_lexers) }
+