Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Version 0.3, using pipes for IPC

  • Loading branch information...
commit d6a8f2af7d20eb71184d6bd7ca60e00501146e88 1 parent adfd47b
@tnm tnm authored
Showing with 13,812 additions and 865 deletions.
  1. +1 −0  .gitignore
  2. +45 −19 README.md
  3. +21 −11 Rakefile
  4. +15 −48 bench.rb
  5. +8 −0 cache-lexers.rb
  6. +0 −14 ext/extconf.rb
  7. +0 −466 ext/pygments.c
  8. BIN  lexers
  9. +3 −6 lib/pygments.rb
  10. +0 −54 lib/pygments/c.rb
  11. +0 −155 lib/pygments/ffi.rb
  12. +343 −0 lib/pygments/mentos.py
  13. +383 −0 lib/pygments/popen.rb
  14. +1 −1  lib/pygments/version.rb
  15. +5 −4 pygments.rb.gemspec
  16. +2,581 −0 test/test_data.c
  17. +514 −0 test/test_data.py
  18. +2,582 −0 test/test_data_generated
  19. +208 −84 test/test_pygments.rb
  20. +0 −1  vendor/.gitignore
  21. +1 −1  vendor/pygments-main/pygments/lexers/_mapping.py
  22. +1 −1  vendor/pygments-main/pygments/lexers/shell.py
  23. +10 −0 vendor/simplejson/.gitignore
  24. +5 −0 vendor/simplejson/.travis.yml
  25. +291 −0 vendor/simplejson/CHANGES.txt
  26. +19 −0 vendor/simplejson/LICENSE.txt
  27. +5 −0 vendor/simplejson/MANIFEST.in
  28. +19 −0 vendor/simplejson/README.rst
  29. +179 −0 vendor/simplejson/conf.py
  30. +628 −0 vendor/simplejson/index.rst
  31. +18 −0 vendor/simplejson/scripts/make_docs.py
  32. +104 −0 vendor/simplejson/setup.py
  33. +510 −0 vendor/simplejson/simplejson/__init__.py
  34. +2,745 −0 vendor/simplejson/simplejson/_speedups.c
  35. +425 −0 vendor/simplejson/simplejson/decoder.py
  36. +567 −0 vendor/simplejson/simplejson/encoder.py
  37. +119 −0 vendor/simplejson/simplejson/ordered_dict.py
  38. +77 −0 vendor/simplejson/simplejson/scanner.py
  39. +67 −0 vendor/simplejson/simplejson/tests/__init__.py
  40. +55 −0 vendor/simplejson/simplejson/tests/test_bigint_as_string.py
  41. +30 −0 vendor/simplejson/simplejson/tests/test_check_circular.py
  42. +66 −0 vendor/simplejson/simplejson/tests/test_decimal.py
  43. +83 −0 vendor/simplejson/simplejson/tests/test_decode.py
  44. +9 −0 vendor/simplejson/simplejson/tests/test_default.py
  45. +67 −0 vendor/simplejson/simplejson/tests/test_dump.py
  46. +46 −0 vendor/simplejson/simplejson/tests/test_encode_basestring_ascii.py
  47. +32 −0 vendor/simplejson/simplejson/tests/test_encode_for_html.py
  48. +34 −0 vendor/simplejson/simplejson/tests/test_errors.py
  49. +91 −0 vendor/simplejson/simplejson/tests/test_fail.py
  50. +19 −0 vendor/simplejson/simplejson/tests/test_float.py
  51. +86 −0 vendor/simplejson/simplejson/tests/test_indent.py
  52. +20 −0 vendor/simplejson/simplejson/tests/test_item_sort_key.py
  53. +121 −0 vendor/simplejson/simplejson/tests/test_namedtuple.py
  54. +76 −0 vendor/simplejson/simplejson/tests/test_pass1.py
  55. +14 −0 vendor/simplejson/simplejson/tests/test_pass2.py
  56. +20 −0 vendor/simplejson/simplejson/tests/test_pass3.py
  57. +67 −0 vendor/simplejson/simplejson/tests/test_recursion.py
  58. +117 −0 vendor/simplejson/simplejson/tests/test_scanstring.py
  59. +42 −0 vendor/simplejson/simplejson/tests/test_separators.py
  60. +20 −0 vendor/simplejson/simplejson/tests/test_speedups.py
  61. +49 −0 vendor/simplejson/simplejson/tests/test_tuple.py
  62. +109 −0 vendor/simplejson/simplejson/tests/test_unicode.py
  63. +39 −0 vendor/simplejson/simplejson/tool.py
View
1  .gitignore
@@ -3,3 +3,4 @@ ext/Makefile
lib/pygments_ext.*
tmp
pkg
+*.pyc
View
64 README.md
@@ -1,14 +1,23 @@
# pygments.rb
-A ruby wrapper for the python [pygments syntax highlighter](http://pygments.org/).
+A Ruby wrapper for the Python [pygments syntax highlighter](http://pygments.org/).
-This library replaces [github/albino](https://github.com/github/albino).
-Instead of shelling out to `pygmentize`, it embeds the python
-interpreter inside ruby via FFI. This avoids the cost of setting up the
-python VM on every invocation and speeds up code highlighting from ruby by 10-15x.
+pygments.rb works by talking over a simple pipe to a long-lived
+Python child process. This library replaces [github/albino](https://github.com/github/albino),
+as well as a version of pygments.rb that used an embedded Python
+interpreter.
+
+Each Ruby process that runs has its own 'personal Python';
+for example, 4 Unicorn workers will have one Python process each.
+If a Python process dies, a new one will be spawned on the next
+pygments.rb request.
## usage
+``` ruby
+require 'pygments'
+```
+
``` ruby
Pygments.highlight(File.read(__FILE__), :lexer => 'ruby')
```
@@ -20,29 +29,34 @@ options hash:
Pygments.highlight('code', :options => {:encoding => 'utf-8'})
```
-To use a formatter other than html, specify it explicitly:
+pygments.rb defaults to using an HTML formatter.
+To use a formatter other than `html`, specify it explicitly
+like so:
``` ruby
Pygments.highlight('code', :formatter => 'bbcode')
Pygments.highlight('code', :formatter => 'terminal')
```
-To generate CSS for html formatted code, use the css method:
+To generate CSS for HTML formatted code, use the `#css` method:
``` ruby
Pygments.css
Pygments.css('.highlight')
```
-To use a custom python installation (like in ArchLinux), tell
-RubyPython where python lives:
+Other Pygments high-level API methods are also available.
+These methods return arrays detailing all the available lexers, formatters,
+and styles.
``` ruby
-RubyPython.configure :python_exe => 'python2.7'
+Pygments.lexers
+Pygments.formatters
+Pygments.styles
```
To use a custom pygments installation, specify the path to
-Pygments.start:
+`Pygments#start`:
``` ruby
Pygments.start("/path/to/pygments")
@@ -50,12 +64,24 @@ Pygments.start("/path/to/pygments")
## benchmarks
- $ ruby -rubygems bench.rb 50
- user system total real
- albino 0.050000 0.050000 12.830000 ( 13.180806)
- pygments::c 1.000000 0.010000 1.010000 ( 1.009348)
- pygments::ffi + reload 11.350000 1.240000 12.590000 ( 12.692320)
- pygments::ffi 1.130000 0.010000 1.140000 ( 1.171589)
-To run `bench.rb`, use a git checkout. The C extension is not included
-in gem releases.
+ $ ruby bench.rb 50
+ Benchmarking....
+ Size: 698 bytes
+ Iterations: 50
+ user system total real
+ pygments popen 0.010000 0.010000 0.020000 ( 0.460370)
+ pygments popen (process already started) 0.010000 0.000000 0.010000 ( 0.272975)
+ pygments popen (process already started 2) 0.000000 0.000000 0.000000 ( 0.273589)
+
+ $ ruby bench.rb 10
+ Benchmarking....
+ Size: 15523 bytes
+ Iterations: 10
+ user system total real
+ pygments popen 0.000000 0.000000 0.000000 ( 0.819419)
+ pygments popen (process already started) 0.010000 0.000000 0.010000 ( 0.676515)
+ pygments popen (process already started 2) 0.000000 0.010000 0.010000 ( 0.674189)
+
+
+
View
32 Rakefile
@@ -1,3 +1,6 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
task :default => :test
# ==========================================================
@@ -11,16 +14,6 @@ Rake::GemPackageTask.new(GEMSPEC) do |pkg|
end
# ==========================================================
-# Ruby Extension
-# ==========================================================
-
-require 'rake/extensiontask'
-Rake::ExtensionTask.new('pygments_ext', GEMSPEC) do |ext|
- ext.ext_dir = 'ext'
-end
-task :build => :compile
-
-# ==========================================================
# Testing
# ==========================================================
@@ -29,7 +22,24 @@ Rake::TestTask.new 'test' do |t|
t.test_files = FileList['test/test_*.rb']
t.ruby_opts = ['-rubygems']
end
-task :test => :build
+
+# ==========================================================
+# Benchmarking
+# ==========================================================
+
+task :bench do
+ sh "ruby bench.rb"
+end
+
+# ==========================================================
+# Cache lexers
+# # ==========================================================
+
+# Write all the lexers to a file for easy lookup
+task :lexers do
+ sh "ruby cache-lexers.rb"
+end
+
# ==========================================================
# Vendor
View
63 bench.rb
@@ -1,55 +1,22 @@
-$:.unshift('lib')
-
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
require 'benchmark'
-require 'pygments/c'
-require 'pygments/ffi'
-require 'rubygems'
-require 'albino'
-num = ARGV[0] ? ARGV[0].to_i : 25
-code = File.read(__FILE__)
+include Benchmark
+# number of iterations
+num = ARGV[0] ? ARGV[0].to_i : 10
-albino, pygments, ffi =
- Albino.new(code, :ruby, :html).colorize,
- Pygments::C.highlight(code, :lexer => 'ruby'),
- Pygments::FFI.highlight(code, :lexer => 'ruby')
+# we can also repeat the code itself
+repeats = ARGV[1] ? ARGV[1].to_i : 1
-unless albino == pygments and pygments == ffi
- raise "incompatible implementations (#{albino.size} != #{pygments.size} != #{ffi.size})"
-end
+code = File.open('test/test_data.py').read.to_s * repeats
-Benchmark.bm(25) do |x|
- x.report('albino') do
- num.times do
- Albino.new(code, :ruby, :html).colorize
- end
- end
- x.report('pygments::c') do
- num.times do
- Pygments::C.highlight(code, :lexer => 'ruby')
- end
- end
- x.report('pygments::ffi + reload') do
- num.times do
- Pygments::FFI.start
- Pygments::FFI.highlight(code, :lexer => 'ruby')
- Pygments::FFI.stop
- end
- end
- Pygments::FFI.start
- x.report('pygments::ffi') do
- num.times do
- Pygments::FFI.highlight(code, :lexer => 'ruby')
- end
- end
-end
+puts "Benchmarking....\n"
+puts "Size: " + code.bytesize.to_s + " bytes\n"
+puts "Iterations: " + num.to_s + "\n"
-__END__
-
-$ ruby -rubygems bench.rb 50
- user system total real
-albino 0.050000 0.050000 12.830000 ( 13.180806)
-pygments::c 1.000000 0.010000 1.010000 ( 1.009348)
-pygments::ffi + reload 11.350000 1.240000 12.590000 ( 12.692320)
-pygments::ffi 1.130000 0.010000 1.140000 ( 1.171589)
+Benchmark.bm(40) do |x|
+ x.report("pygments popen ") { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+ x.report("pygments popen (process already started) ") { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+ x.report("pygments popen (process already started 2) ") { for i in 1..num; Pygments.highlight(code, :lexer => 'python'); end }
+end
View
8 cache-lexers.rb
@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), '/lib/pygments.rb')
+
+# Simple marshalling
+serialized_lexers = Marshal.dump(Pygments.lexers!)
+
+# Write to a file
+File.open("lexers", 'w') { |file| file.write(serialized_lexers) }
+
View
14 ext/extconf.rb
@@ -1,14 +0,0 @@
-require 'mkmf'
-
-python = %w[ 2.7 2.6 2.5 2.4 ].find do |version|
- have_library("python#{version}", 'Py_Initialize', "python#{version}/Python.h")
-end
-
-$CFLAGS << " -Wall "
-
-unless python
- $stderr.puts '*** could not find libpython or Python.h'
-else
- $defs << "-DPYGMENTS_PYTHON_VERSION=#{python.gsub('.','')}"
- create_makefile('pygments_ext')
-end
View
466 ext/pygments.c
@@ -1,466 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <ruby.h>
-
-#if PYGMENTS_PYTHON_VERSION == 24
-#include <python2.4/Python.h>
-#elif PYGMENTS_PYTHON_VERSION == 25
-#include <python2.5/Python.h>
-#elif PYGMENTS_PYTHON_VERSION == 26
-#include <python2.6/Python.h>
-#elif PYGMENTS_PYTHON_VERSION == 27
-#include <python2.7/Python.h>
-#else
-#error Unknown python version
-#endif
-
-#ifdef RUBY_VM
-#include <ruby/st.h>
-#include <ruby/encoding.h>
-#else
-#include <st.h>
-#endif
-
-#ifndef RSTRING_PTR
-#define RSTRING_PTR(str) RSTRING(str)->ptr
-#endif
-#ifndef RSTRING_LEN
-#define RSTRING_LEN(str) RSTRING(str)->len
-#endif
-
-static VALUE mPygments, mPygmentsC;
-static PyObject
- /* modules */
- *pygments,
- *pygments_lexers,
- *pygments_formatters,
- *pygments_styles,
- *pygments_filters,
-
- /* lexer methods */
- *guess_lexer,
- *guess_lexer_for_filename,
- *get_lexer_for_mimetype,
- *get_lexer_for_filename,
- *get_lexer_by_name,
- *get_all_lexers,
-
- /* formatter methods */
- *get_formatter_by_name,
- *get_all_formatters,
-
- /* highlighter methods */
- *highlight,
-
- /* style methods */
- *get_all_styles,
-
- /* filter methods */
- *get_all_filters;
-
-static int
-each_hash_i(VALUE key, VALUE val, VALUE arg)
-{
- Check_Type(key, T_SYMBOL);
-
- PyObject *py = (PyObject*) arg, *py_val = NULL;
- switch (TYPE(val)) {
- case T_NIL:
- return ST_CONTINUE;
- break;
-
- case T_FALSE:
- py_val = Py_False;
- break;
-
- case T_TRUE:
- py_val = Py_True;
- break;
-
- case T_STRING:
- py_val = PyString_FromString(RSTRING_PTR(val));
- break;
-
- default:
- Check_Type(val, T_STRING);
- }
-
- PyDict_SetItemString(py, rb_id2name(SYM2ID(key)), py_val);
-
- return ST_CONTINUE;
-}
-
-static PyObject*
-rb_to_py(VALUE rb)
-{
- PyObject *py = NULL;
-
- switch (TYPE(rb)) {
- case T_HASH:
- py = PyDict_New();
- rb_hash_foreach(rb, each_hash_i, (VALUE)py);
- break;
- }
-
- return py;
-}
-
-static VALUE
-py_to_rb(PyObject *py)
-{
- VALUE rb = Qnil;
- Py_ssize_t len, i;
-
- if (py) {
- if (PyIter_Check(py)) {
- PyObject *item;
- rb = rb_ary_new();
-
- while ((item = PyIter_Next(py))) {
- rb_ary_push(rb, py_to_rb(item));
- Py_DECREF(item);
- }
-
- } else if (PyString_Check(py)) {
- char *data;
-
- if (PyString_AsStringAndSize(py, &data, &len) == 0) {
- rb = rb_str_new(data, len);
- }
-
- } else if (PyTuple_Check(py)) {
- len = PyTuple_Size(py);
-
- PyObject *item;
- rb = rb_ary_new();
-
- for (i=0; i<len; i++) {
- item = PyTuple_GetItem(py, i);
- rb_ary_push(rb, py_to_rb(item));
- }
-
- } else if (PyList_Check(py)) {
- len = PyList_Size(py);
-
- PyObject *item;
- rb = rb_ary_new();
-
- for (i=0; i<len; i++) {
- item = PyList_GetItem(py, i);
- rb_ary_push(rb, py_to_rb(item));
- }
-
- } else if (PyDict_Check(py)) {
- PyObject *key, *val;
- Py_ssize_t pos = 0;
-
- rb = rb_hash_new();
-
- while (PyDict_Next(py, &pos, &key, &val)) {
- rb_hash_aset(rb, py_to_rb(key), py_to_rb(val));
- }
- }
- }
-
- return rb;
-}
-
-static PyObject*
-pygments__lexer_for(VALUE code, VALUE options)
-{
- VALUE filename, mimetype, lexer;
- PyObject *ret = NULL, *args = NULL, *kwargs = NULL;
-
- if (RTEST(code))
- Check_Type(code, T_STRING);
-
- if (RTEST(options)) {
- Check_Type(options, T_HASH);
-
- VALUE kw = rb_hash_aref(options, ID2SYM(rb_intern("options")));
- if (RTEST(kw)) {
- Check_Type(kw, T_HASH);
- kwargs = rb_to_py(kw);
- }
-
- lexer = rb_hash_aref(options, ID2SYM(rb_intern("lexer")));
- filename = rb_hash_aref(options, ID2SYM(rb_intern("filename")));
- mimetype = rb_hash_aref(options, ID2SYM(rb_intern("mimetype")));
-
- if (RTEST(lexer)) {
- Check_Type(lexer, T_STRING);
- args = Py_BuildValue("(s)", RSTRING_PTR(lexer));
- ret = PyObject_Call(get_lexer_by_name, args, kwargs);
-
- } else if (RTEST(mimetype)) {
- Check_Type(mimetype, T_STRING);
- args = Py_BuildValue("(s)", RSTRING_PTR(mimetype));
- ret = PyObject_Call(get_lexer_for_mimetype, args, kwargs);
-
- } else if (RTEST(filename)) {
- Check_Type(filename, T_STRING);
-
- if (RTEST(code)) {
- args = Py_BuildValue("(ss)", RSTRING_PTR(filename), RSTRING_PTR(code));
- ret = PyObject_Call(guess_lexer_for_filename, args, kwargs);
- } else {
- args = Py_BuildValue("(s)", RSTRING_PTR(filename));
- ret = PyObject_Call(get_lexer_for_filename, args, kwargs);
- }
- }
- }
-
- if (ret == NULL && RTEST(code)) {
- args = Py_BuildValue("(s)", RSTRING_PTR(code));
- ret = PyObject_Call(guess_lexer, args, kwargs);
- }
-
- Py_XDECREF(args);
- Py_XDECREF(kwargs);
- PyErr_Clear();
- return ret;
-}
-
-static VALUE
-rb_pygments_lexer_name_for(int argc, VALUE *argv, VALUE self)
-{
- VALUE code = Qnil, options = Qnil;
- VALUE name = Qnil;
- PyObject *lexer = NULL, *aliases = NULL;
-
- int found = rb_scan_args(argc, argv, "11", &code, &options);
-
- if (found > 0) {
- if (found == 1 && TYPE(code) == T_HASH) {
- options = code;
- code = Qnil;
- }
-
- lexer = pygments__lexer_for(code, options);
-
- if (lexer) {
- aliases = PyObject_GetAttrString(lexer, "aliases");
- if (aliases && PyList_Size(aliases) > 0) {
- PyObject *alias = PyList_GetItem(aliases, 0);
- name = rb_str_new2(PyString_AsString(alias));
- }
- }
- }
-
- Py_XDECREF(aliases);
- Py_XDECREF(lexer);
- PyErr_Clear();
- return name;
-}
-
-static VALUE
-rb_pygments_css(int argc, VALUE *argv, VALUE self)
-{
- VALUE css = Qnil, prefix = Qnil, options = Qnil;
- PyObject *args = NULL, *kwargs = NULL, *formatter = NULL;
- int found = rb_scan_args(argc, argv, "02", &prefix, &options);
-
- if (found == 1 && TYPE(prefix) == T_HASH) {
- options = prefix;
- prefix = Qnil;
- }
-
- if (RTEST(prefix))
- Check_Type(prefix, T_STRING);
-
- if (RTEST(options)) {
- Check_Type(options, T_HASH);
- kwargs = rb_to_py(options);
- }
-
- args = Py_BuildValue("(s)", "html");
- formatter = PyObject_Call(get_formatter_by_name, args, kwargs);
- if (formatter) {
- PyObject *styles = PyObject_CallMethod(formatter, "get_style_defs", "(s)", RTEST(prefix) ? RSTRING_PTR(prefix) : "");
- if (styles) {
- css = rb_str_new2(PyString_AsString(styles));
- }
- Py_XDECREF(styles);
- }
-
- Py_XDECREF(args);
- Py_XDECREF(kwargs);
- Py_XDECREF(formatter);
- PyErr_Clear();
-
- return css;
-}
-
-static VALUE
-rb_pygments_highlight(int argc, VALUE *argv, VALUE self)
-{
- PyObject *args = NULL, *kwargs = NULL, *lexer = NULL, *formatter = NULL;
- VALUE code = Qnil, options = Qnil, ret = Qnil, format = Qnil;
- rb_scan_args(argc, argv, "11", &code, &options);
-
- if (RTEST(options)) {
- format = rb_hash_aref(options, ID2SYM(rb_intern("formatter")));
- if (RTEST(format))
- Check_Type(format, T_STRING);
-
- VALUE kw = rb_hash_aref(options, ID2SYM(rb_intern("options")));
- if (RTEST(kw)) {
- Check_Type(kw, T_HASH);
- kwargs = rb_to_py(kw);
- }
- }
-
- lexer = pygments__lexer_for(code, options);
-
- if (lexer) {
- args = Py_BuildValue("(s)", RTEST(format) ? RSTRING_PTR(format) : "html");
- formatter = PyObject_Call(get_formatter_by_name, args, kwargs);
- Py_XDECREF(args);
- args = NULL;
-
- if (formatter) {
- PyObject *input = NULL, *output = NULL;
-
- input = PyUnicode_FromStringAndSize(RSTRING_PTR(code), RSTRING_LEN(code));
- if (input) {
- output = PyObject_CallFunction(highlight, "(OOO)", input, lexer, formatter);
-
- if (output) {
- PyObject *string = PyUnicode_AsEncodedString(output, "utf-8", "strict");
- if (string) {
- Py_ssize_t len;
- char *data;
-
- if (PyString_AsStringAndSize(string, &data, &len) == 0) {
- ret = rb_str_new(data, len);
- }
- }
- Py_XDECREF(string);
- }
- }
-
- Py_XDECREF(output);
- Py_XDECREF(input);
- }
- }
-
- Py_XDECREF(args);
- Py_XDECREF(kwargs);
- Py_XDECREF(lexer);
- PyErr_Clear();
-
-#ifdef RUBY_VM
- if (RTEST(ret))
- rb_funcall(ret, rb_intern("force_encoding"), 1, rb_str_new2("utf-8"));
-#endif
-
- return ret;
-}
-
-static VALUE
-rb_pygments_styles(int argc, VALUE *argv, VALUE self)
-{
- PyObject *styles = PyObject_CallFunction(get_all_styles, "");
- VALUE ret = py_to_rb(styles);
- Py_XDECREF(styles);
- PyErr_Clear();
- return ret;
-}
-
-static VALUE
-rb_pygments_filters(int argc, VALUE *argv, VALUE self)
-{
- PyObject *filters = PyObject_CallFunction(get_all_filters, "");
- VALUE ret = py_to_rb(filters);
- Py_XDECREF(filters);
- PyErr_Clear();
- return ret;
-}
-
-static VALUE
-rb_pygments_lexers(int argc, VALUE *argv, VALUE self)
-{
- PyObject *lexers = PyObject_CallFunction(get_all_lexers, "");
- VALUE ret = py_to_rb(lexers);
- Py_XDECREF(lexers);
- PyErr_Clear();
- return ret;
-}
-
-static VALUE
-rb_pygments_formatters(int argc, VALUE *argv, VALUE self)
-{
- PyObject *formatters = PyObject_CallFunction(get_all_formatters, "");
- PyObject *item;
- VALUE ret = rb_ary_new();
-
- while ((item = PyIter_Next(formatters))) {
- VALUE curr = rb_ary_new();
- rb_ary_push(ret, curr);
-
- rb_ary_push(curr, py_to_rb(PyObject_GetAttrString(item, "__name__")));
- rb_ary_push(curr, py_to_rb(PyObject_GetAttrString(item, "name")));
- rb_ary_push(curr, py_to_rb(PyObject_GetAttrString(item, "aliases")));
-
- Py_DECREF(item);
- }
-
- Py_XDECREF(formatters);
- PyErr_Clear();
- return ret;
-}
-
-#define ENSURE(var, expr) do{ \
- if ((var = (expr)) == NULL) { \
- rb_raise(rb_eRuntimeError, "unable to lookup " # var); \
- } \
-} while(0)
-
-void
-Init_pygments_ext()
-{
- { /* python stuff */
- Py_Initialize();
-
- /* modules */
- ENSURE(pygments, PyImport_ImportModule("pygments"));
- ENSURE(pygments_lexers, PyImport_ImportModule("pygments.lexers"));
- ENSURE(pygments_formatters, PyImport_ImportModule("pygments.formatters"));
- ENSURE(pygments_styles, PyImport_ImportModule("pygments.styles"));
- ENSURE(pygments_filters, PyImport_ImportModule("pygments.filters"));
-
- /* lexer methods */
- ENSURE(guess_lexer, PyObject_GetAttrString(pygments_lexers, "guess_lexer"));
- ENSURE(guess_lexer_for_filename, PyObject_GetAttrString(pygments_lexers, "guess_lexer_for_filename"));
- ENSURE(get_lexer_for_filename, PyObject_GetAttrString(pygments_lexers, "get_lexer_for_filename"));
- ENSURE(get_lexer_for_mimetype, PyObject_GetAttrString(pygments_lexers, "get_lexer_for_mimetype"));
- ENSURE(get_lexer_by_name, PyObject_GetAttrString(pygments_lexers, "get_lexer_by_name"));
- ENSURE(get_all_lexers, PyObject_GetAttrString(pygments_lexers, "get_all_lexers"));
-
- /* formatter methods */
- ENSURE(get_formatter_by_name, PyObject_GetAttrString(pygments_formatters, "get_formatter_by_name"));
- ENSURE(get_all_formatters, PyObject_GetAttrString(pygments_formatters, "get_all_formatters"));
-
- /* highlighter methods */
- ENSURE(highlight, PyObject_GetAttrString(pygments, "highlight"));
-
- /* style methods */
- ENSURE(get_all_styles, PyObject_GetAttrString(pygments_styles, "get_all_styles"));
-
- /* filter methods */
- ENSURE(get_all_filters, PyObject_GetAttrString(pygments_filters, "get_all_filters"));
- }
-
- { /* ruby stuff */
- mPygments = rb_define_module("Pygments");
- mPygmentsC = rb_define_module_under(mPygments, "C");
- rb_define_method(mPygmentsC, "lexer_name_for", rb_pygments_lexer_name_for, -1);
- rb_define_method(mPygmentsC, "css", rb_pygments_css, -1);
- rb_define_method(mPygmentsC, "_highlight", rb_pygments_highlight, -1);
- rb_define_method(mPygmentsC, "styles", rb_pygments_styles, 0);
- rb_define_method(mPygmentsC, "filters", rb_pygments_filters, 0);
- rb_define_method(mPygmentsC, "_lexers", rb_pygments_lexers, 0);
- rb_define_method(mPygmentsC, "_formatters", rb_pygments_formatters, 0);
- }
-}
View
BIN  lexers
Binary file not shown
View
9 lib/pygments.rb
@@ -1,11 +1,8 @@
-# require 'pygments/c'
-require 'pygments/ffi'
+require File.join(File.dirname(__FILE__), 'pygments/popen')
+
module Pygments
- # include Pygments::C
- include Pygments::FFI
+ extend Pygments::Popen
autoload :Lexer, 'pygments/lexer'
-
- extend self
end
View
54 lib/pygments/c.rb
@@ -1,54 +0,0 @@
-module Pygments
- module C
- extend self
-
- def start(python_path = File.expand_path('../../../vendor/pygments-main/', __FILE__))
- ENV['PYTHONPATH'], prev = python_path, ENV['PYTHONPATH']
- require 'pygments_ext'
- @started = true
- ensure
- ENV['PYTHONPATH'] = prev
- end
-
- def stop
- end
-
- def formatters
- start unless @started
-
- _formatters.inject(Hash.new) do |hash, (name, desc, aliases)|
- name.sub!(/Formatter$/,'')
- hash[name] = {
- :name => name,
- :description => desc,
- :aliases => aliases
- }
- hash
- end
- end
-
- def lexers
- start unless @started
-
- _lexers.inject(Hash.new) do |hash, (name, aliases, files, mimes)|
- hash[name] = {
- :description => name,
- :aliases => aliases,
- :filenames => files,
- :mimetypes => mimes
- }
- hash
- end
- end
-
- def highlight(code, opts={})
- start unless @started
-
- out = _highlight(code, opts)
- if opts[:formatter].nil? or opts[:formatter].to_s.downcase == 'html'
- out.gsub!(%r{</pre></div>\Z}, "</pre>\n</div>")
- end
- out
- end
- end
-end
View
155 lib/pygments/ffi.rb
@@ -1,155 +0,0 @@
-require 'rubypython'
-
-module Pygments
- module FFI
- extend self
-
- def start(pygments_path = File.expand_path('../../../vendor/pygments-main/', __FILE__))
- RubyPython.start
- RubyPython.import('pkg_resources') rescue nil
- sys = RubyPython.import('sys')
- sys.path.insert(0, pygments_path)
-
- @modules = [ :lexers, :formatters, :styles, :filters ].inject(Hash.new) do |hash, name|
- hash[name] = RubyPython.import("pygments.#{name}")
- hash
- end
-
- @pygments = RubyPython.import('pygments')
- end
-
- def stop
- RubyPython.stop
- @pygments = nil
- @modules = {}
- end
-
- def formatters
- start unless pygments
- @modules[:formatters].get_all_formatters.to_enum.inject(Hash.new) do |hash, fmt|
- name = fmt.__name__.rubify.sub!(/Formatter$/,'')
-
- hash[name] = {
- :name => name,
- :description => fmt.name.rubify,
- :aliases => fmt.aliases.rubify
- }
- hash
- end
- end
-
- def lexers
- start unless pygments
- @modules[:lexers].get_all_lexers.to_enum.inject(Hash.new) do |hash, lxr|
- lxr = lxr.rubify
- name = lxr.first
-
- hash[name] = {
- :name => name,
- :aliases => lxr[1],
- :filenames => lxr[2],
- :mimetypes => lxr[3]
- }
- hash
- end
- end
-
- def filters
- start unless pygments
- @modules[:filters].get_all_filters.to_enum.map{ |o| o.rubify }
- end
-
- def styles
- start unless pygments
- @modules[:styles].get_all_styles.to_enum.map{ |o| o.rubify }
- end
-
- def css(klass='', opts={})
- if klass.is_a?(Hash)
- opts = klass
- klass = ''
- end
- fmt = formatter_for('html', opts)
- fmt.get_style_defs(klass).rubify
- end
-
- def lexer_name_for(*args)
- lxr = lexer_for(*args)
- lxr.aliases[0].rubify if lxr
- end
-
- def highlight(code, opts={})
- start unless pygments
-
- if code.nil? or code.empty?
- return code
- end
-
- opts[:options] ||= {}
- opts[:options][:outencoding] ||= 'utf-8'
-
- lexer = lexer_for(code, opts)
-
- kwargs = opts[:options] || {}
- fmt_name = (opts[:formatter] || 'html').downcase
- formatter = formatter_for(fmt_name, kwargs)
-
- out = pygments.highlight(code, lexer, formatter)
- str = out.rubify
-
- # ruby's GC will clean these up eventually, but we explicitly
- # decref to avoid unncessary memory/gc pressure.
- [ lexer, formatter, out ].each do |obj|
- obj.pObject.xDecref
- end
-
- str.force_encoding(opts[:options][:outencoding]) if str.respond_to?(:force_encoding)
- if fmt_name == 'html'
- str.gsub!(%r{</pre></div>\Z}, "</pre>\n</div>")
- end
-
- str
- end
-
- private
-
- attr_reader :pygments
-
- def formatter_for(name, opts={})
- start unless pygments
- @modules[:formatters].get_formatter_by_name!(name, opts)
- end
-
- def lexer_for(code, opts={})
- start unless pygments
-
- if code.is_a?(Hash)
- opts = code
- code = nil
- end
-
- mod = @modules[:lexers]
- kwargs = opts[:options] || {}
-
- if name = opts[:lexer]
- mod.get_lexer_by_name!(name, kwargs)
-
- elsif name = opts[:mimetype]
- mod.get_lexer_for_mimetype!(name, kwargs)
-
- elsif name = opts[:filename]
- if code
- mod.guess_lexer_for_filename!(name, code, kwargs)
- else
- mod.get_lexer_for_filename!(name, kwargs)
- end
-
- elsif code
- mod.guess_lexer!(code, kwargs)
-
- else
- nil
- end
- end
- end
-end
View
343 lib/pygments/mentos.py
@@ -0,0 +1,343 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import sys, re, os, signal, resource
+import traceback
+if 'PYGMENTS_PATH' in os.environ:
+ sys.path.insert(0, os.environ['PYGMENTS_PATH'])
+
+dirname = os.path.dirname
+
+base_dir = dirname(dirname(dirname(os.path.abspath(__file__))))
+sys.path.append(base_dir + "/vendor")
+sys.path.append(base_dir + "/vendor/pygments-main")
+sys.path.append(base_dir + "/vendor/simplejson")
+
+import pygments
+from pygments import lexers, formatters, styles, filters
+
+from threading import Lock
+
+try:
+ import json
+except ImportError:
+ import simplejson as json
+
+def _convert_keys(dictionary):
+ if not isinstance(dictionary, dict):
+ return dictionary
+ return dict((str(k), _convert_keys(v))
+ for k, v in dictionary.items())
+
+def _write_error(error):
+ res = {"error": error}
+ out_header = json.dumps(res).encode('utf-8')
+ bits = _get_fixed_bits_from_header(out_header)
+ sys.stdout.write(bits + "\n")
+ sys.stdout.flush()
+ sys.stdout.write(out_header + "\n")
+ sys.stdout.flush()
+ return
+
+def _get_fixed_bits_from_header(out_header):
+ size = len(out_header)
+ return "".join(map(lambda y:str((size>>y)&1), range(32-1, -1, -1)))
+
+def _signal_handler(signal, frame):
+ """
+ Handle the signal given in the first argument, exiting gracefully
+ """
+ sys.exit(0)
+
+class Mentos(object):
+ """
+ Interacts with pygments.rb to provide access to pygments functionality
+ """
+ def __init__(self):
+ pass
+
+ def return_lexer(self, lexer, args, inputs, code=None):
+ """
+ Accepting a variety of possible inputs, return a Lexer object.
+
+ The inputs argument should be a hash with at least one of the following
+ keys:
+
+ - 'lexer' ("python")
+ - 'mimetype' ("text/x-ruby")
+ - 'filename' ("yeaaah.py")
+ - 'code' ("import derp" etc)
+
+ The code guessing method is not especially great. It is advised that
+ clients pass in a literal lexer name whenever possible, which provides
+ the best probability of match (100 percent).
+ """
+
+ if lexer:
+ if inputs:
+ return lexers.get_lexer_by_name(lexer, **inputs)
+ else:
+ return lexers.get_lexer_by_name(lexer)
+
+ if inputs:
+ if 'lexer' in inputs:
+ return lexers.get_lexer_by_name(inputs['lexer'], **inputs)
+
+ elif 'mimetype' in inputs:
+ return lexers.get_lexer_for_mimetype(inputs['mimetype'], **inputs)
+
+ elif 'filename' in inputs:
+ name = inputs['filename']
+
+ # If we have code and a filename, pygments allows us to guess
+ # with both. This is better than just guessing with code.
+ if code:
+ return lexers.guess_lexer_for_filename(name, code, **inputs)
+ else:
+ return lexers.get_lexer_for_filename(name, **inputs)
+
+ # If all we got is code, try anyway.
+ if code:
+ return lexers.guess_lexer(code, **inputs)
+
+ else:
+ _write_error("No lexer")
+
+
+ def highlight_text(self, code, lexer, formatter_name, args, kwargs):
+ """
+ Highlight the relevant code, and return a result string.
+ The default formatter is html, but alternate formatters can be passed in via
+ the formatter_name argument. Additional paramters can be passed as args
+ or kwargs.
+ """
+ # Default to html if we don't have the formatter name.
+ if formatter_name:
+ _format_name = str(formatter_name)
+ else:
+ _format_name = "html"
+
+ # Return a lexer object
+ lexer = self.return_lexer(lexer, args, kwargs, code)
+
+ # Make sure we sucessfuly got a lexer
+ if lexer:
+ formatter = pygments.formatters.get_formatter_by_name(str.lower(_format_name), **kwargs)
+
+ # Do the damn thing.
+ res = pygments.highlight(code, lexer, formatter)
+
+ return res
+
+ else:
+ _write_error("No lexer")
+
+ def get_data(self, method, lexer, args, kwargs, text=None):
+ """
+ Based on the method argument, determine the action we'd like pygments
+ to do. Then return the data generated from pygments.
+ """
+ if kwargs:
+ formatter_name = kwargs.get("formatter", None)
+ opts = kwargs.get("options", {})
+
+ # Ensure there's a 'method' key before proceeeding
+ if method:
+ res = None
+
+ # Now check what that method is. For the get methods, pygments
+ # itself returns generators, so we make them lists so we can serialize
+ # easier.
+ if method == 'get_all_styles':
+ res = json.dumps(list(pygments.styles.get_all_styles()))
+
+ elif method == 'get_all_filters':
+ res = json.dumps(list(pygments.filters.get_all_filters()))
+
+ elif method == 'get_all_lexers':
+ res = json.dumps(list(pygments.lexers.get_all_lexers()))
+
+ elif method == 'get_all_formatters':
+ res = [ [ft.__name__, ft.name, ft.aliases] for ft in pygments.formatters.get_all_formatters() ]
+ res = json.dumps(res)
+
+ elif method == 'highlight':
+ try:
+ text = text.decode('utf-8')
+ except UnicodeDecodeError:
+ # The text may already be encoded
+ text = text
+ res = self.highlight_text(text, lexer, formatter_name, args, _convert_keys(opts))
+
+ elif method == 'css':
+ kwargs = _convert_keys(kwargs)
+ fmt = pygments.formatters.get_formatter_by_name(args[0], **kwargs)
+ res = fmt.get_style_defs(args[1])
+
+ elif method == 'lexer_name_for':
+ lexer = self.return_lexer(None, args, kwargs, text)
+
+ if lexer:
+ # We don't want the Lexer itself, just the name.
+ # Take the first alias.
+ res = lexer.aliases[0]
+
+ else:
+ _write_error("No lexer")
+
+ else:
+ _write_error("No lexer")
+
+ return res
+
+
+ def _send_data(self, res, method):
+
+ # Base header. We'll build on this, adding keys as necessary.
+ base_header = {"method": method}
+
+ res_bytes = len(res) + 1
+ base_header["bytes"] = res_bytes
+
+ out_header = json.dumps(base_header).encode('utf-8')
+
+ # Following the protocol, send over a fixed size represenation of the
+ # size of the JSON header
+ bits = _get_fixed_bits_from_header(out_header)
+
+ # Send it to Rubyland
+ sys.stdout.write(bits + "\n")
+ sys.stdout.flush()
+
+ # Send the header.
+ sys.stdout.write(out_header + "\n")
+ sys.stdout.flush()
+
+ # Finally, send the result
+ sys.stdout.write(res + "\n")
+ sys.stdout.flush()
+
+
+ def _get_ids(self, text):
+ start_id = text[:8]
+ end_id = text[-8:]
+ return start_id, end_id
+
+ def _check_and_return_text(self, text, start_id, end_id):
+
+ # Sanity check.
+ id_regex = re.compile('[A-Z]{8}')
+
+ if not id_regex.match(start_id) and not id_regex.match(end_id):
+ _write_error("ID check failed. Not an ID.")
+
+ if not start_id == end_id:
+ _write_error("ID check failed. ID's did not match.")
+
+ # Passed the sanity check. Remove the id's and return
+ text = text[10:-10]
+ return text
+
+ def _parse_header(self, header):
+ method = header["method"]
+ args = header.get("args", [])
+ kwargs = header.get("kwargs", {})
+ lexer = kwargs.get("lexer", None)
+ return (method, args, kwargs, lexer)
+
+ def start(self):
+ """
+ Main loop, waiting for inputs on stdin. When it gets some data,
+ it goes to work.
+
+ mentos exposes most of the "High-level API" of pygments. It always
+ expects and requires a JSON header of metadata. If there is data to be
+ pygmentized, this header will be followed by the text to be pygmentized.
+
+ The header is of form:
+ { "method": "highlight", "args": [], "kwargs": {"arg1": "v"}, "bytes": 128, "fd": "8"}
+ """
+ lock = Lock()
+
+ while True:
+ # The loop begins by reading off a simple 32-arity string
+ # representing an integer of 32 bits. This is the length of
+ # our JSON header.
+ size = sys.stdin.read(32)
+
+ lock.acquire()
+
+ try:
+ # Read from stdin the amount of bytes we were told to expect.
+ header_bytes = int(size, 2)
+
+ # Sanity check the size
+ size_regex = re.compile('[0-1]{32}')
+ if not size_regex.match(size):
+ _write_error("Size received is not valid.")
+
+ line = sys.stdin.read(header_bytes)
+
+ header = json.loads(line)
+
+ method, args, kwargs, lexer = self._parse_header(header)
+ _bytes = 0
+
+ if lexer:
+ lexer = str(lexer)
+
+ # Read more bytes if necessary
+ if kwargs:
+ _bytes = kwargs.get("bytes", 0)
+
+ # Read up to the given number bytes (possibly 0)
+ text = sys.stdin.read(_bytes)
+
+ # Sanity check the return.
+ if method == 'highlight':
+ start_id, end_id = self._get_ids(text)
+ text = self._check_and_return_text(text, start_id, end_id)
+
+ # Get the actual data from pygments.
+ res = self.get_data(method, lexer, args, kwargs, text)
+
+ # Put back the sanity check values.
+ if method == "highlight":
+ res = start_id + " " + res + " " + end_id
+
+ self._send_data(res, method)
+
+ except:
+ tb = traceback.format_exc()
+ _write_error(tb)
+
+ finally:
+ lock.release()
+
+def main():
+
+ # Signal handlers to trap signals.
+ signal.signal(signal.SIGINT, _signal_handler)
+ signal.signal(signal.SIGTERM, _signal_handler)
+ signal.signal(signal.SIGHUP, _signal_handler)
+
+ mentos = Mentos()
+
+ # close fd's inherited from the ruby parent
+ maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+ if maxfd == resource.RLIM_INFINITY:
+ maxfd = 65536
+
+ for fd in range(3, maxfd):
+ try:
+ os.close(fd)
+ except:
+ pass
+
+ mentos.start()
+
+if __name__ == "__main__":
+ main()
+
+
+
View
383 lib/pygments/popen.rb
@@ -0,0 +1,383 @@
+# coding: utf-8
+require 'posix/spawn'
+require 'yajl'
+require 'timeout'
+require 'logger'
+require 'time'
+
+# Error class
+class MentosError < IOError
+end
+
+# Pygments provides access to the Pygments library via a pipe and a long-running
+# Python process.
+module Pygments
+ module Popen
+ include POSIX::Spawn
+ extend self
+
+ # Get things started by opening a pipe to mentos (the freshmaker), a
+ # Python process that talks to the Pygments library. We'll talk back and
+ # forth across this pipe.
+ def start(pygments_path = File.expand_path('../../../vendor/pygments-main/', __FILE__))
+ begin
+ @log = Logger.new(ENV['MENTOS_LOG'] ||= '/dev/null')
+ @log.level = Logger::INFO
+ @log.datetime_format = "%Y-%m-%d %H:%M "
+ rescue
+ @log = Logger.new('/dev/null')
+ end
+
+ ENV['PYGMENTS_PATH'] = pygments_path
+
+ # Make sure we kill off the child when we're done
+ at_exit { stop "Exiting" }
+
+ # A pipe to the mentos python process. #popen4 gives us
+ # the pid and three IO objects to write and read.
+ @pid, @in, @out, @err = popen4(File.expand_path('../mentos.py', __FILE__))
+ @log.info "[#{Time.now.iso8601}] Starting pid #{@pid.to_s} with fd #{@out.to_i.to_s}."
+ end
+
+ # Stop the child process by issuing a kill -9.
+ #
+ # We then call waitpid() with the pid, which waits for that particular
+ # child and reaps it.
+ #
+ # kill() can set errno to ESRCH if, for some reason, the file
+ # is gone; regardless the final outcome of this method
+ # will be to set our @pid variable to nil.
+ #
+ # Technically, kill() can also fail with EPERM or EINVAL (wherein
+ # the signal isn't sent); but we have permissions, and
+ # we're not doing anything invalid here.
+ def stop(reason)
+ if @pid
+ begin
+ Process.kill('KILL', @pid)
+ Process.waitpid(@pid)
+ rescue Errno::ESRCH, Errno::ECHILD
+ end
+ end
+ @log.info "[#{Time.now.iso8601}] Killing pid: #{@pid.to_s}. Reason: #{reason}"
+ @pid = nil
+ end
+
+ # Check for a @pid variable, and then hit `kill -0` with the pid to
+ # check if the pid is still in the process table. If this function
+ # gives us an ENOENT or ESRCH, we can also safely return false (no process
+ # to worry about). Defensively, if EPERM is raised, in a odd/rare
+ # dying process situation (e.g., mentos is checking on the pid of a dead
+ # process and the pid has already been re-used) we'll want to raise
+ # that as a more informative Mentos exception.
+ #
+ # Returns true if the child is alive.
+ def alive?
+ return true if @pid && Process.kill(0, @pid)
+ false
+ rescue Errno::ENOENT, Errno::ESRCH
+ false
+ rescue Errno::EPERM
+ raise MentosError, "EPERM checking if child process is alive."
+ end
+
+ # Public: Get an array of available Pygments formatters
+ #
+ # Returns an array of formatters.
+ def formatters
+ mentos(:get_all_formatters).inject(Hash.new) do | hash, (name, desc, aliases) |
+ # Remove the long-winded and repetitive 'Formatter' suffix
+ name.sub!(/Formatter$/, '')
+ hash[name] = {
+ :name => name,
+ :description => desc,
+ :aliases => aliases
+ }
+ hash
+ end
+ end
+
+
+ # Public: Get all lexers from a serialized array. This avoids needing to spawn
+ # mentos when it's not really needed (e.g,. one-off jobs, loading the Rails env, etc).
+ #
+ # Should be preferred to #lexers!
+ #
+ # Returns an array of lexers
+ def lexers
+ begin
+ lexer_file = File.expand_path('../../../lexers', __FILE__)
+ raw = File.open(lexer_file, "r").read
+ Marshal.load(raw)
+ rescue Errno::ENOENT
+ raise MentosError, "Error loading lexer file. Was it created and vendored?"
+ end
+ end
+
+ # Public: Get back all available lexers from mentos itself
+ #
+ # Returns an array of lexers
+ def lexers!
+ mentos(:get_all_lexers).inject(Hash.new) do |hash, lxr|
+ name = lxr[0]
+ hash[name] = {
+ :name => name,
+ :aliases => lxr[1],
+ :filenames => lxr[2],
+ :mimetypes => lxr[3]
+ }
+ hash
+ end
+ end
+
+ # Public: Return an array of all available filters
+ def filters
+ mentos(:get_all_filters)
+ end
+
+ # Public: Return an array of all available styles
+ def styles
+ mentos(:get_all_styles)
+ end
+
+ # Public: Return css for highlighted code
+ def css(klass='', opts={})
+ if klass.is_a?(Hash)
+ opts = klass
+ klass = ''
+ end
+ mentos(:css, ['html', klass], opts)
+ end
+
+ # Public: Return the name of a lexer.
+ def lexer_name_for(*args)
+ # Pop off the last arg if it's a hash, which becomes our opts
+ if args.last.is_a?(Hash)
+ opts = args.pop
+ else
+ opts = {}
+ end
+
+ mentos(:lexer_name_for, args, opts)
+ end
+
+ # Public: Highlight code.
+ #
+ # Takes a first-position argument of the code to be highlighted, and a
+ # second-position hash of various arguments specifiying highlighting properties.
+ def highlight(code, opts={})
+ # If the caller didn't give us any code, we have nothing to do,
+ # so return right away.
+ return code if code.nil? || code.empty?
+
+ # Callers pass along options in the hash
+ opts[:options] ||= {}
+
+ # Default to utf-8 for the output encoding, if not given.
+ opts[:options][:outencoding] ||= 'utf-8'
+
+ # Get back the string from mentos and force encoding if we can
+ str = mentos(:highlight, nil, opts, code)
+ str.force_encoding(opts[:options][:outencoding]) if str.respond_to?(:force_encoding)
+ str
+ end
+
+ private
+
+ # Our 'rpc'-ish request to mentos. Requires a method name, and then optional
+ # args, kwargs, code.
+ def mentos(method, args=[], kwargs={}, original_code=nil)
+ # Open the pipe if necessary
+ start unless alive?
+
+ begin
+ # Timeout requests that take too long.
+ timeout_time = 8
+
+ Timeout::timeout(timeout_time) do
+ # For sanity checking on both sides of the pipe when highlighting, we prepend and
+ # append an id. mentos checks that these are 8 character ids and that they match.
+ # It then returns the id's back to Rubyland.
+ id = (0...8).map{65.+(rand(25)).chr}.join
+ code = add_ids(original_code, id) if original_code
+
+ # Add metadata to the header and generate it.
+ if code
+ bytesize = code.bytesize
+ else
+ bytesize = 0
+ end
+
+ kwargs.freeze
+ kwargs = kwargs.merge("fd" => @out.to_i, "id" => id, "bytes" => bytesize)
+ out_header = Yajl.dump(:method => method, :args => args, :kwargs => kwargs)
+
+ # Get the size of the header itself and write that.
+ bits = get_fixed_bits_from_header(out_header)
+ @in.write(bits)
+
+ # mentos is now waiting for the header, and, potentially, code.
+ write_data(out_header, code)
+
+ # mentos will now return data to us. First it sends the header.
+ header = get_header
+
+ # Now handle the header, any read any more data required.
+ res = handle_header_and_return(header, id)
+
+ # Finally, return what we got.
+ return_result(res, method)
+ end
+ rescue Timeout::Error
+ # If we timeout, we need to clear out the pipe and start over.
+ @log.error "[#{Time.now.iso8601}] Timeout on a mentos #{method} call"
+ stop "Timeout on mentos #{method} call."
+ end
+
+ rescue Errno::EPIPE, EOFError
+ stop "EPIPE"
+ raise MentosError, "EPIPE"
+ end
+
+
+ # Based on the header we receive, determine if we need
+ # to read more bytes, and read those bytes if necessary.
+ #
+ # Then, do a sanity check wih the ids.
+ #
+ # Returns a result — either highlighted text or metadata.
+ def handle_header_and_return(header, id)
+ if header
+ header = header_to_json(header)
+ bytes = header["bytes"]
+
+ # Read more bytes (the actual response body)
+ res = @out.read(bytes.to_i)
+
+ if header["method"] == "highlight"
+ # Make sure we have a result back; else consider this an error.
+ if res.nil?
+ @log.warn "[#{Time.now.iso8601}] No highlight result back from mentos."
+ stop "No highlight result back from mentos."
+ raise MentosError, "No highlight result back from mentos."
+ end
+
+ # Remove the newline from Python
+ res = res[0..-2]
+ @log.info "[#{Time.now.iso8601}] Highlight in process."
+
+ # Get the id's
+ start_id = res[0..7]
+ end_id = res[-8..-1]
+
+ # Sanity check.
+ if not (start_id == id and end_id == id)
+ @log.error "[#{Time.now.iso8601}] ID's did not match. Aborting."
+ stop "ID's did not match. Aborting."
+ raise MentosError, "ID's did not match. Aborting. " + res.to_s
+ else
+ # We're good. Remove the padding
+ res = res[10..-11]
+ @log.info "[#{Time.now.iso8601}] Highlighting complete."
+ res
+ end
+ end
+ res
+ else
+ @log.error "[#{Time.now.iso8601}] No header data back."
+ stop "No header data back."
+ raise MentosError, "No header received back."
+ end
+ end
+
+ # With the code, prepend the id (with two spaces to avoid escaping weirdness if
+ # the following text starts with a slash (like terminal code), and append the
+ # id, with two padding also. This means we are sending over the 8 characters +
+ # code + 8 characters.
+ def add_ids(code, id)
+ code.freeze
+ code = id + " #{code} #{id}"
+ code
+ end
+
+ # Write data to mentos, the Python Process.
+ #
+ # Returns nothing.
+ def write_data(out_header, code=nil)
+ @in.write(out_header)
+ @log.info "[#{Time.now.iso8601}] Out header: #{out_header.to_s}"
+ @in.write(code) if code
+ end
+
+ # Sanity check for size (32-arity of 0's and 1's)
+ def size_check(size)
+ size_regex = /[0-1]{32}/
+ if size_regex.match(size)
+ true
+ else
+ false
+ end
+ end
+
+ # Read the header via the pipe.
+ #
+ # Returns a header.
+ def get_header
+ begin
+ size = @out.read(33)
+ size = size[0..-2]
+
+ # Sanity check the size
+ if not size_check(size)
+ @log.error "[#{Time.now.iso8601}] Size returned from Mentos invalid."
+ stop "Size returned from Mentos invalid."
+ raise MentosError, "Size returned from Mentos invalid."
+ end
+
+ # Read the amount of bytes we should be expecting. We first
+ # convert the string of bits into an integer.
+ header_bytes = size.to_s.to_i(2) + 1
+ @log.info "[#{Time.now.iso8601}] Size in: #{size.to_s} (#{header_bytes.to_s})"
+ @out.read(header_bytes)
+ rescue
+ @log.error "[#{Time.now.iso8601}] Failed to get header."
+ stop "Failed to get header."
+ raise MentosError, "Failed to get header."
+ end
+ end
+
+ # Return the final result for the API. Return Ruby objects for the methods that
+ # want them, text otherwise.
+ def return_result(res, method)
+ unless method == :lexer_name_for || method == :highlight || method == :css
+ res = Yajl.load(res, :symbolize_keys => true)
+ end
+ res = res[0..-2]
+ end
+
+ # Convert a text header into JSON for easy access.
+ def header_to_json(header)
+ @log.info "[#{Time.now.iso8601}] In header: #{header.to_s} "
+ header = Yajl.load(header)
+
+ if header["error"]
+ # Raise this as a Ruby exception of the MentosError class.
+ # Stop so we don't leave the pipe in an inconsistent state.
+ @log.error "[#{Time.now.iso8601}] Failed to convert header to JSON."
+ stop header["error"]
+ raise MentosError, header["error"]
+ else
+ header
+ end
+ end
+
+ def get_fixed_bits_from_header(out_header)
+ size = out_header.bytesize
+
+ # Fixed 32 bits to represent the int. We return a string
+ # represenation: e.g, "00000000000000000000000000011110"
+ Array.new(32) { |i| size[i] }.reverse!.join
+ end
+ end
+end
+
View
2  lib/pygments/version.rb
@@ -1,3 +1,3 @@
module Pygments
- VERSION = '0.2.13'
+ VERSION = '0.3.0'
end
View
9 pygments.rb.gemspec
@@ -5,16 +5,17 @@ Gem::Specification.new do |s|
s.version = Pygments::VERSION
s.summary = 'pygments wrapper for ruby'
- s.description = 'pygments.rb exposes the pygments syntax highlighter via embedded python'
+ s.description = 'pygments.rb exposes the pygments syntax highlighter to Ruby'
s.homepage = 'http://github.com/tmm1/pygments.rb'
s.has_rdoc = false
- s.authors = ['Aman Gupta']
+ s.authors = ['Aman Gupta', 'Ted Nyman']
s.email = ['aman@tmm1.net']
- s.add_dependency 'rubypython', '~> 0.5.3'
- s.add_development_dependency 'rake-compiler', '0.7.6'
+ s.add_dependency 'yajl-ruby', '~> 1.1.0'
+ s.add_dependency 'posix-spawn', '~> 0.3.6'
+ s.add_development_dependency 'rake-compiler', '~> 0.7.6'
# s.extensions = ['ext/extconf.rb']
s.require_paths = ['lib']
View
2,581 test/test_data.c
@@ -0,0 +1,2581 @@
+/*
+ * Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * * Neither the name of Redis nor the names of its contributors may be used
+ * to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "redis.h"
+#include "slowlog.h"
+#include "bio.h"
+
+#include <time.h>
+#include <signal.h>
+#include <sys/wait.h>
+#include <errno.h>
+#include <assert.h>
+#include <ctype.h>
+#include <stdarg.h>
+#include <arpa/inet.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <sys/uio.h>
+#include <limits.h>
+#include <float.h>
+#include <math.h>
+#include <sys/resource.h>
+#include <sys/utsname.h>
+
+/* Our shared "common" objects */
+
+struct sharedObjectsStruct shared;
+
+/* Global vars that are actually used as constants. The following double
+ * values are used for double on-disk serialization, and are initialized
+ * at runtime to avoid strange compiler optimizations. */
+
+double R_Zero, R_PosInf, R_NegInf, R_Nan;
+
+/*================================= Globals ================================= */
+
+/* Global vars */
+struct redisServer server; /* server global state */
+struct redisCommand *commandTable;
+
+/* Our command table.
+ *
+ * Every entry is composed of the following fields:
+ *
+ * name: a string representing the command name.
+ * function: pointer to the C function implementing the command.
+ * arity: number of arguments, it is possible to use -N to say >= N
+ * sflags: command flags as string. See below for a table of flags.
+ * flags: flags as bitmask. Computed by Redis using the 'sflags' field.
+ * get_keys_proc: an optional function to get key arguments from a command.
+ * This is only used when the following three fields are not
+ * enough to specify what arguments are keys.
+ * first_key_index: first argument that is a key
+ * last_key_index: last argument that is a key
+ * key_step: step to get all the keys from first to last argument. For instance
+ * in MSET the step is two since arguments are key,val,key,val,...
+ * microseconds: microseconds of total execution time for this command.
+ * calls: total number of calls of this command.
+ *
+ * The flags, microseconds and calls fields are computed by Redis and should
+ * always be set to zero.
+ *
+ * Command flags are expressed using strings where every character represents
+ * a flag. Later the populateCommandTable() function will take care of
+ * populating the real 'flags' field using this characters.
+ *
+ * This is the meaning of the flags:
+ *
+ * w: write command (may modify the key space).
+ * r: read command (will never modify the key space).
+ * m: may increase memory usage once called. Don't allow if out of memory.
+ * a: admin command, like SAVE or SHUTDOWN.
+ * p: Pub/Sub related command.
+ * f: force replication of this command, regarless of server.dirty.
+ * s: command not allowed in scripts.
+ * R: random command. Command is not deterministic, that is, the same command
+ * with the same arguments, with the same key space, may have different
+ * results. For instance SPOP and RANDOMKEY are two random commands.
+ * S: Sort command output array if called from script, so that the output
+ * is deterministic.
+ */
+struct redisCommand redisCommandTable[] = {
+ {"get",getCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"set",setCommand,3,"wm",0,noPreloadGetKeys,1,1,1,0,0},
+ {"setnx",setnxCommand,3,"wm",0,noPreloadGetKeys,1,1,1,0,0},
+ {"setex",setexCommand,4,"wm",0,noPreloadGetKeys,1,1,1,0,0},
+ {"psetex",psetexCommand,4,"wm",0,noPreloadGetKeys,1,1,1,0,0},
+ {"append",appendCommand,3,"wm",0,NULL,1,1,1,0,0},
+ {"strlen",strlenCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"del",delCommand,-2,"w",0,noPreloadGetKeys,1,-1,1,0,0},
+ {"exists",existsCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"setbit",setbitCommand,4,"wm",0,NULL,1,1,1,0,0},
+ {"getbit",getbitCommand,3,"r",0,NULL,1,1,1,0,0},
+ {"setrange",setrangeCommand,4,"wm",0,NULL,1,1,1,0,0},
+ {"getrange",getrangeCommand,4,"r",0,NULL,1,1,1,0,0},
+ {"substr",getrangeCommand,4,"r",0,NULL,1,1,1,0,0},
+ {"incr",incrCommand,2,"wm",0,NULL,1,1,1,0,0},
+ {"decr",decrCommand,2,"wm",0,NULL,1,1,1,0,0},
+ {"mget",mgetCommand,-2,"r",0,NULL,1,-1,1,0,0},
+ {"rpush",rpushCommand,-3,"wm",0,NULL,1,1,1,0,0},
+ {"lpush",lpushCommand,-3,"wm",0,NULL,1,1,1,0,0},
+ {"rpushx",rpushxCommand,3,"wm",0,NULL,1,1,1,0,0},
+ {"lpushx",lpushxCommand,3,"wm",0,NULL,1,1,1,0,0},
+ {"linsert",linsertCommand,5,"wm",0,NULL,1,1,1,0,0},
+ {"rpop",rpopCommand,2,"w",0,NULL,1,1,1,0,0},
+ {"lpop",lpopCommand,2,"w",0,NULL,1,1,1,0,0},
+ {"brpop",brpopCommand,-3,"ws",0,NULL,1,1,1,0,0},
+ {"brpoplpush",brpoplpushCommand,4,"wms",0,NULL,1,2,1,0,0},
+ {"blpop",blpopCommand,-3,"ws",0,NULL,1,-2,1,0,0},
+ {"llen",llenCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"lindex",lindexCommand,3,"r",0,NULL,1,1,1,0,0},
+ {"lset",lsetCommand,4,"wm",0,NULL,1,1,1,0,0},
+ {"lrange",lrangeCommand,4,"r",0,NULL,1,1,1,0,0},
+ {"ltrim",ltrimCommand,4,"w",0,NULL,1,1,1,0,0},
+ {"lrem",lremCommand,4,"w",0,NULL,1,1,1,0,0},
+ {"rpoplpush",rpoplpushCommand,3,"wm",0,NULL,1,2,1,0,0},
+ {"sadd",saddCommand,-3,"wm",0,NULL,1,1,1,0,0},
+ {"srem",sremCommand,-3,"w",0,NULL,1,1,1,0,0},
+ {"smove",smoveCommand,4,"w",0,NULL,1,2,1,0,0},
+ {"sismember",sismemberCommand,3,"r",0,NULL,1,1,1,0,0},
+ {"scard",scardCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"spop",spopCommand,2,"wRs",0,NULL,1,1,1,0,0},
+ {"srandmember",srandmemberCommand,2,"rR",0,NULL,1,1,1,0,0},
+ {"sinter",sinterCommand,-2,"rS",0,NULL,1,-1,1,0,0},
+ {"sinterstore",sinterstoreCommand,-3,"wm",0,NULL,1,-1,1,0,0},
+ {"sunion",sunionCommand,-2,"rS",0,NULL,1,-1,1,0,0},
+ {"sunionstore",sunionstoreCommand,-3,"wm",0,NULL,1,-1,1,0,0},
+ {"sdiff",sdiffCommand,-2,"rS",0,NULL,1,-1,1,0,0},
+ {"sdiffstore",sdiffstoreCommand,-3,"wm",0,NULL,1,-1,1,0,0},
+ {"smembers",sinterCommand,2,"rS",0,NULL,1,1,1,0,0},
+ {"zadd",zaddCommand,-4,"wm",0,NULL,1,1,1,0,0},
+ {"zincrby",zincrbyCommand,4,"wm",0,NULL,1,1,1,0,0},
+ {"zrem",zremCommand,-3,"w",0,NULL,1,1,1,0,0},
+ {"zremrangebyscore",zremrangebyscoreCommand,4,"w",0,NULL,1,1,1,0,0},
+ {"zremrangebyrank",zremrangebyrankCommand,4,"w",0,NULL,1,1,1,0,0},
+ {"zunionstore",zunionstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
+ {"zinterstore",zinterstoreCommand,-4,"wm",0,zunionInterGetKeys,0,0,0,0,0},
+ {"zrange",zrangeCommand,-4,"r",0,NULL,1,1,1,0,0},
+ {"zrangebyscore",zrangebyscoreCommand,-4,"r",0,NULL,1,1,1,0,0},
+ {"zrevrangebyscore",zrevrangebyscoreCommand,-4,"r",0,NULL,1,1,1,0,0},
+ {"zcount",zcountCommand,4,"r",0,NULL,1,1,1,0,0},
+ {"zrevrange",zrevrangeCommand,-4,"r",0,NULL,1,1,1,0,0},
+ {"zcard",zcardCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"zscore",zscoreCommand,3,"r",0,NULL,1,1,1,0,0},
+ {"zrank",zrankCommand,3,"r",0,NULL,1,1,1,0,0},
+ {"zrevrank",zrevrankCommand,3,"r",0,NULL,1,1,1,0,0},
+ {"hset",hsetCommand,4,"wm",0,NULL,1,1,1,0,0},
+ {"hsetnx",hsetnxCommand,4,"wm",0,NULL,1,1,1,0,0},
+ {"hget",hgetCommand,3,"r",0,NULL,1,1,1,0,0},
+ {"hmset",hmsetCommand,-4,"wm",0,NULL,1,1,1,0,0},
+ {"hmget",hmgetCommand,-3,"r",0,NULL,1,1,1,0,0},
+ {"hincrby",hincrbyCommand,4,"wm",0,NULL,1,1,1,0,0},
+ {"hincrbyfloat",hincrbyfloatCommand,4,"wm",0,NULL,1,1,1,0,0},
+ {"hdel",hdelCommand,-3,"w",0,NULL,1,1,1,0,0},
+ {"hlen",hlenCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"hkeys",hkeysCommand,2,"rS",0,NULL,1,1,1,0,0},
+ {"hvals",hvalsCommand,2,"rS",0,NULL,1,1,1,0,0},
+ {"hgetall",hgetallCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"hexists",hexistsCommand,3,"r",0,NULL,1,1,1,0,0},
+ {"incrby",incrbyCommand,3,"wm",0,NULL,1,1,1,0,0},
+ {"decrby",decrbyCommand,3,"wm",0,NULL,1,1,1,0,0},
+ {"incrbyfloat",incrbyfloatCommand,3,"wm",0,NULL,1,1,1,0,0},
+ {"getset",getsetCommand,3,"wm",0,NULL,1,1,1,0,0},
+ {"mset",msetCommand,-3,"wm",0,NULL,1,-1,2,0,0},
+ {"msetnx",msetnxCommand,-3,"wm",0,NULL,1,-1,2,0,0},
+ {"randomkey",randomkeyCommand,1,"rR",0,NULL,0,0,0,0,0},
+ {"select",selectCommand,2,"r",0,NULL,0,0,0,0,0},
+ {"move",moveCommand,3,"w",0,NULL,1,1,1,0,0},
+ {"rename",renameCommand,3,"w",0,renameGetKeys,1,2,1,0,0},
+ {"renamenx",renamenxCommand,3,"w",0,renameGetKeys,1,2,1,0,0},
+ {"expire",expireCommand,3,"w",0,NULL,1,1,1,0,0},
+ {"expireat",expireatCommand,3,"w",0,NULL,1,1,1,0,0},
+ {"pexpire",pexpireCommand,3,"w",0,NULL,1,1,1,0,0},
+ {"pexpireat",pexpireatCommand,3,"w",0,NULL,1,1,1,0,0},
+ {"keys",keysCommand,2,"rS",0,NULL,0,0,0,0,0},
+ {"dbsize",dbsizeCommand,1,"r",0,NULL,0,0,0,0,0},
+ {"auth",authCommand,2,"rs",0,NULL,0,0,0,0,0},
+ {"ping",pingCommand,1,"r",0,NULL,0,0,0,0,0},
+ {"echo",echoCommand,2,"r",0,NULL,0,0,0,0,0},
+ {"save",saveCommand,1,"ars",0,NULL,0,0,0,0,0},
+ {"bgsave",bgsaveCommand,1,"ar",0,NULL,0,0,0,0,0},
+ {"bgrewriteaof",bgrewriteaofCommand,1,"ar",0,NULL,0,0,0,0,0},
+ {"shutdown",shutdownCommand,-1,"ar",0,NULL,0,0,0,0,0},
+ {"lastsave",lastsaveCommand,1,"r",0,NULL,0,0,0,0,0},
+ {"type",typeCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"multi",multiCommand,1,"rs",0,NULL,0,0,0,0,0},
+ {"exec",execCommand,1,"s",0,NULL,0,0,0,0,0},
+ {"discard",discardCommand,1,"rs",0,NULL,0,0,0,0,0},
+ {"sync",syncCommand,1,"ars",0,NULL,0,0,0,0,0},
+ {"replconf",replconfCommand,-1,"ars",0,NULL,0,0,0,0,0},
+ {"flushdb",flushdbCommand,1,"w",0,NULL,0,0,0,0,0},
+ {"flushall",flushallCommand,1,"w",0,NULL,0,0,0,0,0},
+ {"sort",sortCommand,-2,"wmS",0,NULL,1,1,1,0,0},
+ {"info",infoCommand,-1,"r",0,NULL,0,0,0,0,0},
+ {"monitor",monitorCommand,1,"ars",0,NULL,0,0,0,0,0},
+ {"ttl",ttlCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"pttl",pttlCommand,2,"r",0,NULL,1,1,1,0,0},
+ {"persist",persistCommand,2,"w",0,NULL,1,1,1,0,0},
+ {"slaveof",slaveofCommand,3,"as",0,NULL,0,0,0,0,0},
+ {"debug",debugCommand,-2,"as",0,NULL,0,0,0,0,0},
+ {"config",configCommand,-2,"ar",0,NULL,0,0,0,0,0},
+ {"subscribe",subscribeCommand,-2,"rps",0,NULL,0,0,0,0,0},
+ {"unsubscribe",unsubscribeCommand,-1,"rps",0,NULL,0,0,0,0,0},
+ {"psubscribe",psubscribeCommand,-2,"rps",0,NULL,0,0,0,0,0},
+ {"punsubscribe",punsubscribeCommand,-1,"rps",0,NULL,0,0,0,0,0},
+ {"publish",publishCommand,3,"pf",0,NULL,0,0,0,0,0},
+ {"watch",watchCommand,-2,"rs",0,noPreloadGetKeys,1,-1,1,0,0},
+ {"unwatch",unwatchCommand,1,"rs",0,NULL,0,0,0,0,0},
+ {"cluster",clusterCommand,-2,"ar",0,NULL,0,0,0,0,0},
+ {"restore",restoreCommand,4,"awm",0,NULL,1,1,1,0,0},
+ {"migrate",migrateCommand,6,"aw",0,NULL,0,0,0,0,0},
+ {"asking",askingCommand,1,"r",0,NULL,0,0,0,0,0},
+ {"dump",dumpCommand,2,"ar",0,NULL,1,1,1,0,0},
+ {"object",objectCommand,-2,"r",0,NULL,2,2,2,0,0},
+ {"client",clientCommand,-2,"ar",0,NULL,0,0,0,0,0},
+ {"eval",evalCommand,-3,"s",0,zunionInterGetKeys,0,0,0,0,0},
+ {"evalsha",evalShaCommand,-3,"s",0,zunionInterGetKeys,0,0,0,0,0},
+ {"slowlog",slowlogCommand,-2,"r",0,NULL,0,0,0,0,0},
+ {"script",scriptCommand,-2,"ras",0,NULL,0,0,0,0,0},
+ {"time",timeCommand,1,"rR",0,NULL,0,0,0,0,0},
+ {"bitop",bitopCommand,-4,"wm",0,NULL,2,-1,1,0,0},
+ {"bitcount",bitcountCommand,-2,"r",0,NULL,1,1,1,0,0}
+};
+
+/*============================ Utility functions ============================ */
+
+/* Low level logging. To use only for very big messages, otherwise
+ * redisLog() is to prefer. */
+void redisLogRaw(int level, const char *msg) {
+ const int syslogLevelMap[] = { LOG_DEBUG, LOG_INFO, LOG_NOTICE, LOG_WARNING };
+ const char *c = ".-*#";
+ FILE *fp;
+ char buf[64];
+ int rawmode = (level & REDIS_LOG_RAW);
+
+ level &= 0xff; /* clear flags */
+ if (level < server.verbosity) return;
+
+ fp = (server.logfile == NULL) ? stdout : fopen(server.logfile,"a");
+ if (!fp) return;
+
+ if (rawmode) {
+ fprintf(fp,"%s",msg);
+ } else {
+ int off;
+ struct timeval tv;
+
+ gettimeofday(&tv,NULL);
+ off = strftime(buf,sizeof(buf),"%d %b %H:%M:%S.",localtime(&tv.tv_sec));
+ snprintf(buf+off,sizeof(buf)-off,"%03d",(int)tv.tv_usec/1000);
+ fprintf(fp,"[%d] %s %c %s\n",(int)getpid(),buf,c[level],msg);
+ }
+ fflush(fp);
+
+ if (server.logfile) fclose(fp);
+
+ if (server.syslog_enabled) syslog(syslogLevelMap[level], "%s", msg);
+}
+
+/* Like redisLogRaw() but with printf-alike support. This is the funciton that
+ * is used across the code. The raw version is only used in order to dump
+ * the INFO output on crash. */
+void redisLog(int level, const char *fmt, ...) {
+ va_list ap;
+ char msg[REDIS_MAX_LOGMSG_LEN];
+
+ if ((level&0xff) < server.verbosity) return;
+
+ va_start(ap, fmt);
+ vsnprintf(msg, sizeof(msg), fmt, ap);
+ va_end(ap);
+
+ redisLogRaw(level,msg);
+}
+
+/* Log a fixed message without printf-alike capabilities, in a way that is
+ * safe to call from a signal handler.
+ *
+ * We actually use this only for signals that are not fatal from the point
+ * of view of Redis. Signals that are going to kill the server anyway and
+ * where we need printf-alike features are served by redisLog(). */
+void redisLogFromHandler(int level, const char *msg) {
+ int fd;
+ char buf[64];
+
+ if ((level&0xff) < server.verbosity ||
+ (server.logfile == NULL && server.daemonize)) return;
+ fd = server.logfile ?
+ open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644) :
+ STDOUT_FILENO;
+ if (fd == -1) return;
+ ll2string(buf,sizeof(buf),getpid());
+ if (write(fd,"[",1) == -1) goto err;