Permalink
Browse files

Version 0.3, using pipes for IPC

  • Loading branch information...
1 parent adfd47b commit d6a8f2af7d20eb71184d6bd7ca60e00501146e88 @tnm tnm committed Sep 25, 2012
Showing with 13,812 additions and 865 deletions.
  1. +1 −0 .gitignore
  2. +45 −19 README.md
  3. +21 −11 Rakefile
  4. +15 −48 bench.rb
  5. +8 −0 cache-lexers.rb
  6. +0 −14 ext/extconf.rb
  7. +0 −466 ext/pygments.c
  8. BIN lexers
  9. +3 −6 lib/pygments.rb
  10. +0 −54 lib/pygments/c.rb
  11. +0 −155 lib/pygments/ffi.rb
  12. +343 −0 lib/pygments/mentos.py
  13. +383 −0 lib/pygments/popen.rb
  14. +1 −1 lib/pygments/version.rb
  15. +5 −4 pygments.rb.gemspec
  16. +2,581 −0 test/test_data.c
  17. +514 −0 test/test_data.py
  18. +2,582 −0 test/test_data_generated
  19. +208 −84 test/test_pygments.rb
  20. +0 −1 vendor/.gitignore
  21. +1 −1 vendor/pygments-main/pygments/lexers/_mapping.py
  22. +1 −1 vendor/pygments-main/pygments/lexers/shell.py
  23. +10 −0 vendor/simplejson/.gitignore
  24. +5 −0 vendor/simplejson/.travis.yml
  25. +291 −0 vendor/simplejson/CHANGES.txt
  26. +19 −0 vendor/simplejson/LICENSE.txt
  27. +5 −0 vendor/simplejson/MANIFEST.in
  28. +19 −0 vendor/simplejson/README.rst
  29. +179 −0 vendor/simplejson/conf.py
  30. +628 −0 vendor/simplejson/index.rst
  31. +18 −0 vendor/simplejson/scripts/make_docs.py
  32. +104 −0 vendor/simplejson/setup.py
  33. +510 −0 vendor/simplejson/simplejson/__init__.py
  34. +2,745 −0 vendor/simplejson/simplejson/_speedups.c
  35. +425 −0 vendor/simplejson/simplejson/decoder.py
  36. +567 −0 vendor/simplejson/simplejson/encoder.py
  37. +119 −0 vendor/simplejson/simplejson/ordered_dict.py
  38. +77 −0 vendor/simplejson/simplejson/scanner.py
  39. +67 −0 vendor/simplejson/simplejson/tests/__init__.py
  40. +55 −0 vendor/simplejson/simplejson/tests/test_bigint_as_string.py
  41. +30 −0 vendor/simplejson/simplejson/tests/test_check_circular.py
  42. +66 −0 vendor/simplejson/simplejson/tests/test_decimal.py
  43. +83 −0 vendor/simplejson/simplejson/tests/test_decode.py
  44. +9 −0 vendor/simplejson/simplejson/tests/test_default.py
  45. +67 −0 vendor/simplejson/simplejson/tests/test_dump.py
  46. +46 −0 vendor/simplejson/simplejson/tests/test_encode_basestring_ascii.py
  47. +32 −0 vendor/simplejson/simplejson/tests/test_encode_for_html.py
  48. +34 −0 vendor/simplejson/simplejson/tests/test_errors.py
  49. +91 −0 vendor/simplejson/simplejson/tests/test_fail.py
  50. +19 −0 vendor/simplejson/simplejson/tests/test_float.py
  51. +86 −0 vendor/simplejson/simplejson/tests/test_indent.py
  52. +20 −0 vendor/simplejson/simplejson/tests/test_item_sort_key.py
  53. +121 −0 vendor/simplejson/simplejson/tests/test_namedtuple.py
  54. +76 −0 vendor/simplejson/simplejson/tests/test_pass1.py
  55. +14 −0 vendor/simplejson/simplejson/tests/test_pass2.py
  56. +20 −0 vendor/simplejson/simplejson/tests/test_pass3.py
  57. +67 −0 vendor/simplejson/simplejson/tests/test_recursion.py
  58. +117 −0 vendor/simplejson/simplejson/tests/test_scanstring.py
  59. +42 −0 vendor/simplejson/simplejson/tests/test_separators.py
  60. +20 −0 vendor/simplejson/simplejson/tests/test_speedups.py
  61. +49 −0 vendor/simplejson/simplejson/tests/test_tuple.py
  62. +109 −0 vendor/simplejson/simplejson/tests/test_unicode.py
  63. +39 −0 vendor/simplejson/simplejson/tool.py
View
@@ -3,3 +3,4 @@ ext/Makefile
lib/pygments_ext.*
tmp
pkg
+*.pyc
View
@@ -1,14 +1,23 @@
# pygments.rb
-A ruby wrapper for the python [pygments syntax highlighter](http://pygments.org/).
+A Ruby wrapper for the Python [pygments syntax highlighter](http://pygments.org/).
-This library replaces [github/albino](https://github.com/github/albino).
-Instead of shelling out to `pygmentize`, it embeds the python
-interpreter inside ruby via FFI. This avoids the cost of setting up the
-python VM on every invocation and speeds up code highlighting from ruby by 10-15x.
+pygments.rb works by talking over a simple pipe to a long-lived
+Python child process. This library replaces [github/albino](https://github.com/github/albino),
+as well as a version of pygments.rb that used an embedded Python
+interpreter.
+
+Each Ruby process that runs has its own 'personal Python';
+for example, 4 Unicorn workers will have one Python process each.
+If a Python process dies, a new one will be spawned on the next
+pygments.rb request.
## usage
+``` ruby
+require 'pygments'
+```
+
``` ruby
Pygments.highlight(File.read(__FILE__), :lexer => 'ruby')
```
@@ -20,42 +29,59 @@ options hash:
Pygments.highlight('code', :options => {:encoding => 'utf-8'})
```
-To use a formatter other than html, specify it explicitly:
+pygments.rb defaults to using an HTML formatter.
+To use a formatter other than `html`, specify it explicitly
+like so:
``` ruby
Pygments.highlight('code', :formatter => 'bbcode')
Pygments.highlight('code', :formatter => 'terminal')
```
-To generate CSS for html formatted code, use the css method:
+To generate CSS for HTML formatted code, use the `#css` method:
``` ruby
Pygments.css
Pygments.css('.highlight')
```
-To use a custom python installation (like in ArchLinux), tell
-RubyPython where python lives:
+Other Pygments high-level API methods are also available.
+These methods return arrays detailing all the available lexers, formatters,
+and styles.
``` ruby
-RubyPython.configure :python_exe => 'python2.7'
+Pygments.lexers
+Pygments.formatters
+Pygments.styles
```
To use a custom pygments installation, specify the path to
-Pygments.start:
+`Pygments#start`:
``` ruby
Pygments.start("/path/to/pygments")
```
## benchmarks
- $ ruby -rubygems bench.rb 50
- user system total real
- albino 0.050000 0.050000 12.830000 ( 13.180806)
- pygments::c 1.000000 0.010000 1.010000 ( 1.009348)
- pygments::ffi + reload 11.350000 1.240000 12.590000 ( 12.692320)
- pygments::ffi 1.130000 0.010000 1.140000 ( 1.171589)
-To run `bench.rb`, use a git checkout. The C extension is not included
-in gem releases.
+ $ ruby bench.rb 50
+ Benchmarking....
+ Size: 698 bytes
+ Iterations: 50
+ user system total real
+ pygments popen 0.010000 0.010000 0.020000 ( 0.460370)
+ pygments popen (process already started) 0.010000 0.000000 0.010000 ( 0.272975)
+ pygments popen (process already started 2) 0.000000 0.000000 0.000000 ( 0.273589)
+
+ $ ruby bench.rb 10
+ Benchmarking....
+ Size: 15523 bytes
+ Iterations: 10
+ user system total real
+ pygments popen 0.000000 0.000000 0.000000 ( 0.819419)
+ pygments popen (process already started) 0.010000 0.000000 0.010000 ( 0.676515)
+ pygments popen (process already started 2) 0.000000 0.010000 0.010000 ( 0.674189)
+
+
+
View
@@ -1,3 +1,6 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
task :default => :test
# ==========================================================
@@ -11,16 +14,6 @@ Rake::GemPackageTask.new(GEMSPEC) do |pkg|
end
# ==========================================================
-# Ruby Extension
-# ==========================================================
-
-require 'rake/extensiontask'
-Rake::ExtensionTask.new('pygments_ext', GEMSPEC) do |ext|
- ext.ext_dir = 'ext'
-end
-task :build => :compile
-
-# ==========================================================
# Testing
# ==========================================================
@@ -29,7 +22,24 @@ Rake::TestTask.new 'test' do |t|
t.test_files = FileList['test/test_*.rb']
t.ruby_opts = ['-rubygems']
end
-task :test => :build
+
+# ==========================================================
+# Benchmarking
+# ==========================================================
+
+task :bench do
+ sh "ruby bench.rb"
+end
+
+# ==========================================================
+# Cache lexers
+# # ==========================================================
+
+# Write all the lexers to a file for easy lookup
+task :lexers do
+ sh "ruby cache-lexers.rb"
+end
+
# ==========================================================
# Vendor
View
@@ -1,55 +1,22 @@
-$:.unshift('lib')
-
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
require 'benchmark'
-require 'pygments/c'
-require 'pygments/ffi'
-require 'rubygems'
-require 'albino'
-num = ARGV[0] ? ARGV[0].to_i : 25
-code = File.read(__FILE__)
+include Benchmark
+# number of iterations
+num = ARGV[0] ? ARGV[0].to_i : 10
-albino, pygments, ffi =
- Albino.new(code, :ruby, :html).colorize,
- Pygments::C.highlight(code, :lexer => 'ruby'),
- Pygments::FFI.highlight(code, :lexer => 'ruby')
+# we can also repeat the code itself
+repeats = ARGV[1] ? ARGV[1].to_i : 1
-unless albino == pygments and pygments == ffi
- raise "incompatible implementations (#{albino.size} != #{pygments.size} != #{ffi.size})"
-end
+code = File.open('test/test_data.py').read.to_s * repeats
-Benchmark.bm(25) do |x|
- x.report('albino') do
- num.times do
- Albino.new(code, :ruby, :html).colorize
- end
- end
- x.report('pygments::c') do
- num.times do
- Pygments::C.highlight(code, :lexer => 'ruby')
- end
- end
- x.report('pygments::ffi + reload') do
- num.times do
- Pygments::FFI.start
- Pygments::FFI.highlight(code, :lexer => 'ruby')
- Pygments::FFI.stop
- end
- end
- Pygments::FFI.start
- x.report('pygments::ffi') do
- num.times do
- Pygments::FFI.highlight(code, :lexer => 'ruby')
- end
- end
-end
+puts "Benchmarking....\n"
+puts "Size: " + code.bytesize.to_s + " bytes\n"
+puts "Iterations: " + num.to_s + "\n"
-__END__
-
-$ ruby -rubygems bench.rb 50
- user system total real
-albino 0.050000 0.050000 12.830000 ( 13.180806)
-pygments::c 1.000000 0.010000 1.010000 ( 1.009348)
-pygments::ffi + reload 11.350000 1.240000 12.590000 ( 12.692320)
-pygments::ffi 1.130000 0.010000 1.140000 ( 1.171589)
+Benchmark.bm(40) do |x|
+ x.report("pygments popen ") { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+ x.report("pygments popen (process already started) ") { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+ x.report("pygments popen (process already started 2) ") { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+end
View
@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
+
+# Simple marshalling
+serialized_lexers = Marshal.dump(Pygments.lexers!)
+
+# Write to a file
+File.open("lexers", 'w') { |file| file.write(serialized_lexers) }
+
View
@@ -1,14 +0,0 @@
-require 'mkmf'
-
-python = %w[ 2.7 2.6 2.5 2.4 ].find do |version|
- have_library("python#{version}", 'Py_Initialize', "python#{version}/Python.h")
-end
-
-$CFLAGS << " -Wall "
-
-unless python
- $stderr.puts '*** could not find libpython or Python.h'
-else
- $defs << "-DPYGMENTS_PYTHON_VERSION=#{python.gsub('.','')}"
- create_makefile('pygments_ext')
-end
Oops, something went wrong.

0 comments on commit d6a8f2a

Please sign in to comment.