Pipe #33

Closed
wants to merge 24 commits into
from
Commits
Jump to file
+13,811 −866
Split
View
@@ -1,5 +1,4 @@
Gemfile.lock
-ext/Makefile
-lib/pygments_ext.*
tmp
pkg
+*.pyc
View
@@ -1,14 +1,23 @@
# pygments.rb
-A ruby wrapper for the python [pygments syntax highlighter](http://pygments.org/).
+A Ruby wrapper for the Python [pygments syntax highlighter](http://pygments.org/).
-This library replaces [github/albino](https://github.com/github/albino).
-Instead of shelling out to `pygmentize`, it embeds the python
-interpreter inside ruby via FFI. This avoids the cost of setting up the
-python VM on every invocation and speeds up code highlighting from ruby by 10-15x.
+pygments.rb works by talking over a simple pipe to a long-lived
+Python child process. This library replaces [github/albino](https://github.com/github/albino),
+as well as a version of pygments.rb that used an embedded Python
+interpreter.
+
+Each Ruby process that runs has its own 'personal Python';
+for example, 4 Unicorn workers will have one Python process each.
+If a Python process dies, a new one will be spawned on the next
+pygments.rb request.
## usage
+``` ruby
+require 'pygments'
+```
+
``` ruby
Pygments.highlight(File.read(__FILE__), :lexer => 'ruby')
```
@@ -20,42 +29,59 @@ options hash:
Pygments.highlight('code', :options => {:encoding => 'utf-8'})
```
-To use a formatter other than html, specify it explicitly:
+pygments.rb defaults to using an HTML formatter.
+To use a formatter other than `html`, specify it explicitly
+like so:
``` ruby
Pygments.highlight('code', :formatter => 'bbcode')
Pygments.highlight('code', :formatter => 'terminal')
```
-To generate CSS for html formatted code, use the css method:
+To generate CSS for HTML formatted code, use the `#css` method:
``` ruby
Pygments.css
Pygments.css('.highlight')
```
-To use a custom python installation (like in ArchLinux), tell
-RubyPython where python lives:
+Other Pygments high-level API methods are also available.
+These methods return arrays detailing all the available lexers, formatters,
+and styles.
``` ruby
-RubyPython.configure :python_exe => 'python2.7'
+Pygments.lexers
+Pygments.formatters
+Pygments.styles
```
To use a custom pygments installation, specify the path to
-Pygments.start:
+`Pygments#start`:
``` ruby
Pygments.start("/path/to/pygments")
```
## benchmarks
- $ ruby -rubygems bench.rb 50
- user system total real
- albino 0.050000 0.050000 12.830000 ( 13.180806)
- pygments::c 1.000000 0.010000 1.010000 ( 1.009348)
- pygments::ffi + reload 11.350000 1.240000 12.590000 ( 12.692320)
- pygments::ffi 1.130000 0.010000 1.140000 ( 1.171589)
-To run `bench.rb`, use a git checkout. The C extension is not included
-in gem releases.
+ $ ruby bench.rb 50
+ Benchmarking....
+ Size: 698 bytes
+ Iterations: 50
+ user system total real
+ pygments popen 0.010000 0.010000 0.020000 ( 0.460370)
+ pygments popen (process already started) 0.010000 0.000000 0.010000 ( 0.272975)
+ pygments popen (process already started 2) 0.000000 0.000000 0.000000 ( 0.273589)
+
+ $ ruby bench.rb 10
+ Benchmarking....
+ Size: 15523 bytes
+ Iterations: 10
+ user system total real
+ pygments popen 0.000000 0.000000 0.000000 ( 0.819419)
+ pygments popen (process already started) 0.010000 0.000000 0.010000 ( 0.676515)
+ pygments popen (process already started 2) 0.000000 0.010000 0.010000 ( 0.674189)
+
+
+
View
@@ -1,3 +1,6 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
task :default => :test
# ==========================================================
@@ -11,16 +14,6 @@ Rake::GemPackageTask.new(GEMSPEC) do |pkg|
end
# ==========================================================
-# Ruby Extension
-# ==========================================================
-
-require 'rake/extensiontask'
-Rake::ExtensionTask.new('pygments_ext', GEMSPEC) do |ext|
- ext.ext_dir = 'ext'
-end
-task :build => :compile
-
-# ==========================================================
# Testing
# ==========================================================
@@ -29,7 +22,24 @@ Rake::TestTask.new 'test' do |t|
t.test_files = FileList['test/test_*.rb']
t.ruby_opts = ['-rubygems']
end
-task :test => :build
+
+# ==========================================================
+# Benchmarking
+# ==========================================================
+
+task :bench do
+ sh "ruby bench.rb"
+end
+
+# ==========================================================
+# Cache lexers
+# # ==========================================================
+
+# Write all the lexers to a file for easy lookup
+task :lexers do
+ sh "ruby cache-lexers.rb"
+end
+
# ==========================================================
# Vendor
View
@@ -1,55 +1,22 @@
-$:.unshift('lib')
-
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
require 'benchmark'
-require 'pygments/c'
-require 'pygments/ffi'
-require 'rubygems'
-require 'albino'
-num = ARGV[0] ? ARGV[0].to_i : 25
-code = File.read(__FILE__)
+include Benchmark
+# number of iterations
+num = ARGV[0] ? ARGV[0].to_i : 10
-albino, pygments, ffi =
- Albino.new(code, :ruby, :html).colorize,
- Pygments::C.highlight(code, :lexer => 'ruby'),
- Pygments::FFI.highlight(code, :lexer => 'ruby')
+# we can also repeat the code itself
+repeats = ARGV[1] ? ARGV[1].to_i : 1
-unless albino == pygments and pygments == ffi
- raise "incompatible implementations (#{albino.size} != #{pygments.size} != #{ffi.size})"
-end
+code = File.open('test/test_data.py').read.to_s * repeats
-Benchmark.bm(25) do |x|
- x.report('albino') do
- num.times do
- Albino.new(code, :ruby, :html).colorize
- end
- end
- x.report('pygments::c') do
- num.times do
- Pygments::C.highlight(code, :lexer => 'ruby')
- end
- end
- x.report('pygments::ffi + reload') do
- num.times do
- Pygments::FFI.start
- Pygments::FFI.highlight(code, :lexer => 'ruby')
- Pygments::FFI.stop
- end
- end
- Pygments::FFI.start
- x.report('pygments::ffi') do
- num.times do
- Pygments::FFI.highlight(code, :lexer => 'ruby')
- end
- end
-end
+puts "Benchmarking....\n"
+puts "Size: " + code.bytesize.to_s + " bytes\n"
+puts "Iterations: " + num.to_s + "\n"
-__END__
-
-$ ruby -rubygems bench.rb 50
- user system total real
-albino 0.050000 0.050000 12.830000 ( 13.180806)
-pygments::c 1.000000 0.010000 1.010000 ( 1.009348)
-pygments::ffi + reload 11.350000 1.240000 12.590000 ( 12.692320)
-pygments::ffi 1.130000 0.010000 1.140000 ( 1.171589)
+Benchmark.bm(40) do |x|
+ x.report("pygments popen ") { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+ x.report("pygments popen (process already started) ") { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+ x.report("pygments popen (process already started 2) ") { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+end
View
@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
+
+# Simple marshalling
+serialized_lexers = Marshal.dump(Pygments.lexers!)
+
+# Write to a file
+File.open("lexers", 'w') { |file| file.write(serialized_lexers) }
+
View
@@ -1,14 +0,0 @@
-require 'mkmf'
-
-python = %w[ 2.7 2.6 2.5 2.4 ].find do |version|
- have_library("python#{version}", 'Py_Initialize', "python#{version}/Python.h")
-end
-
-$CFLAGS << " -Wall "
-
-unless python
- $stderr.puts '*** could not find libpython or Python.h'
-else
- $defs << "-DPYGMENTS_PYTHON_VERSION=#{python.gsub('.','')}"
- create_makefile('pygments_ext')
-end
Oops, something went wrong.