Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

wow, it actually works.

  • Loading branch information...
commit 9bc9075f6b2cca3a0329e7cb88d64012d54847bb 0 parents
Aman Gupta tmm1 authored
5 .gitignore
@@ -0,0 +1,5 @@
+Gemfile.lock
+ext/Makefile
+lib/pygments_ext.*
+tmp
+pkg
2  Gemfile
@@ -0,0 +1,2 @@
+source :rubygems
+gemspec
3  README.md
@@ -0,0 +1,3 @@
+# pygments.rb
+
+a ruby wrapper for the pygments syntax highlighter via embedded python.
31 Rakefile
@@ -0,0 +1,31 @@
+task :default => :test
+
+# ==========================================================
+# Packaging
+# ==========================================================
+
+GEMSPEC = eval(File.read('pygments.rb.gemspec'))
+
+require 'rake/gempackagetask'
+Rake::GemPackageTask.new(GEMSPEC) do |pkg|
+end
+
+# ==========================================================
+# Ruby Extension
+# ==========================================================
+
+require 'rake/extensiontask'
+Rake::ExtensionTask.new('pygments_ext', GEMSPEC) do |ext|
+ ext.ext_dir = 'ext'
+end
+task :build => :compile
+
+# ==========================================================
+# Testing
+# ==========================================================
+
+require 'rake/testtask'
+Rake::TestTask.new 'test' do |t|
+ t.test_files = FileList['test/test_*.rb']
+end
+task :test => :build
14 ext/extconf.rb
@@ -0,0 +1,14 @@
+require 'mkmf'
+
+python = %w[ 2.7 2.6 2.5 2.4 ].find do |version|
+ have_library("python#{version}", 'Py_Initialize', "python#{version}/Python.h")
+end
+
+$CFLAGS << " -Wall "
+
+unless python
+ $stderr.puts '*** could not find libpython or Python.h'
+else
+ $CFLAGS << " -I/usr/include/python#{python} "
+ create_makefile('pygments_ext')
+end
454 ext/pygments.c
@@ -0,0 +1,454 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <ruby.h>
+#include <Python.h>
+
+#ifdef RUBY_VM
+#include <ruby/st.h>
+#include <ruby/encoding.h>
+#else
+#include <st.h>
+#endif
+
+#ifndef RSTRING_PTR
+#define RSTRING_PTR(str) RSTRING(str)->ptr
+#endif
+#ifndef RSTRING_LEN
+#define RSTRING_LEN(str) RSTRING(str)->len
+#endif
+
+static VALUE mPygments;
+static PyObject
+ /* modules */
+ *pygments,
+ *pygments_lexers,
+ *pygments_formatters,
+ *pygments_styles,
+ *pygments_filters,
+
+ /* lexer methods */
+ *guess_lexer,
+ *guess_lexer_for_filename,
+ *get_lexer_for_mimetype,
+ *get_lexer_for_filename,
+ *get_lexer_by_name,
+ *get_all_lexers,
+
+ /* formatter methods */
+ *get_formatter_by_name,
+ *get_all_formatters,
+
+ /* highlighter methods */
+ *highlight,
+
+ /* style methods */
+ *get_all_styles,
+
+ /* filter methods */
+ *get_all_filters;
+
+static int
+each_hash_i(VALUE key, VALUE val, VALUE arg)
+{
+ Check_Type(key, T_SYMBOL);
+
+ PyObject *py = (PyObject*) arg, *py_val = NULL;
+ switch (TYPE(val)) {
+ case T_NIL:
+ return ST_CONTINUE;
+ break;
+
+ case T_FALSE:
+ py_val = Py_False;
+ break;
+
+ case T_TRUE:
+ py_val = Py_True;
+ break;
+
+ case T_STRING:
+ py_val = PyString_FromString(RSTRING_PTR(val));
+ break;
+
+ default:
+ Check_Type(val, T_STRING);
+ }
+
+ PyDict_SetItemString(py, rb_id2name(SYM2ID(key)), py_val);
+
+ return ST_CONTINUE;
+}
+
+static PyObject*
+rb_to_py(VALUE rb)
+{
+ PyObject *py = NULL;
+
+ switch (TYPE(rb)) {
+ case T_HASH:
+ py = PyDict_New();
+ rb_hash_foreach(rb, each_hash_i, (VALUE)py);
+ break;
+ }
+
+ return py;
+}
+
+static VALUE
+py_to_rb(PyObject *py)
+{
+ VALUE rb = Qnil;
+ Py_ssize_t len, i;
+
+ if (py) {
+ if (PyIter_Check(py)) {
+ PyObject *item;
+ rb = rb_ary_new();
+
+ while ((item = PyIter_Next(py))) {
+ rb_ary_push(rb, py_to_rb(item));
+ Py_DECREF(item);
+ }
+
+ } else if (PyString_Check(py)) {
+ char *data;
+
+ if (PyString_AsStringAndSize(py, &data, &len) == 0) {
+ rb = rb_str_new(data, len);
+ }
+
+ } else if (PyTuple_Check(py)) {
+ len = PyTuple_Size(py);
+
+ PyObject *item;
+ rb = rb_ary_new();
+
+ for (i=0; i<len; i++) {
+ item = PyTuple_GetItem(py, i);
+ rb_ary_push(rb, py_to_rb(item));
+ }
+
+ } else if (PyList_Check(py)) {
+ len = PyList_Size(py);
+
+ PyObject *item;
+ rb = rb_ary_new();
+
+ for (i=0; i<len; i++) {
+ item = PyList_GetItem(py, i);
+ rb_ary_push(rb, py_to_rb(item));
+ }
+
+ } else if (PyDict_Check(py)) {
+ PyObject *key, *val;
+ Py_ssize_t pos = 0;
+
+ rb = rb_hash_new();
+
+ while (PyDict_Next(py, &pos, &key, &val)) {
+ rb_hash_aset(rb, py_to_rb(key), py_to_rb(val));
+ }
+ }
+ }
+
+ return rb;
+}
+
+static PyObject*
+pygments__lexer_for(VALUE code, VALUE options)
+{
+ VALUE filename, mimetype, lexer;
+ PyObject *ret = NULL, *args = NULL, *kwargs = NULL;
+
+ if (RTEST(code))
+ Check_Type(code, T_STRING);
+
+ if (RTEST(options)) {
+ Check_Type(options, T_HASH);
+
+ VALUE kw = rb_hash_aref(options, ID2SYM(rb_intern("options")));
+ if (RTEST(kw)) {
+ Check_Type(kw, T_HASH);
+ kwargs = rb_to_py(kw);
+ }
+
+ lexer = rb_hash_aref(options, ID2SYM(rb_intern("lexer")));
+ filename = rb_hash_aref(options, ID2SYM(rb_intern("filename")));
+ mimetype = rb_hash_aref(options, ID2SYM(rb_intern("mimetype")));
+
+ if (RTEST(lexer)) {
+ Check_Type(lexer, T_STRING);
+ args = Py_BuildValue("(s)", RSTRING_PTR(lexer));
+ ret = PyObject_Call(get_lexer_by_name, args, kwargs);
+
+ } else if (RTEST(mimetype)) {
+ Check_Type(mimetype, T_STRING);
+ args = Py_BuildValue("(s)", RSTRING_PTR(mimetype));
+ ret = PyObject_Call(get_lexer_for_mimetype, args, kwargs);
+
+ } else if (RTEST(filename)) {
+ Check_Type(filename, T_STRING);
+
+ if (RTEST(code)) {
+ args = Py_BuildValue("(ss)", RSTRING_PTR(filename), RSTRING_PTR(code));
+ ret = PyObject_Call(guess_lexer_for_filename, args, kwargs);
+ } else {
+ args = Py_BuildValue("(s)", RSTRING_PTR(filename));
+ ret = PyObject_Call(get_lexer_for_filename, args, kwargs);
+ }
+ }
+ }
+
+ if (ret == NULL && RTEST(code)) {
+ args = Py_BuildValue("(s)", RSTRING_PTR(code));
+ ret = PyObject_Call(guess_lexer, args, kwargs);
+ }
+
+ Py_XDECREF(args);
+ Py_XDECREF(kwargs);
+ PyErr_Clear();
+ return ret;
+}
+
+static VALUE
+rb_pygments_lexer_name_for(int argc, VALUE *argv, VALUE self)
+{
+ VALUE code = Qnil, options = Qnil;
+ VALUE name = Qnil;
+ PyObject *lexer = NULL, *aliases = NULL;
+
+ int found = rb_scan_args(argc, argv, "11", &code, &options);
+
+ if (found > 0) {
+ if (found == 1 && TYPE(code) == T_HASH) {
+ options = code;
+ code = Qnil;
+ }
+
+ lexer = pygments__lexer_for(code, options);
+
+ if (lexer) {
+ aliases = PyObject_GetAttrString(lexer, "aliases");
+ if (aliases && PyList_Size(aliases) > 0) {
+ PyObject *alias = PyList_GetItem(aliases, 0);
+ name = rb_str_new2(PyString_AsString(alias));
+ }
+ }
+ }
+
+ Py_XDECREF(aliases);
+ Py_XDECREF(lexer);
+ PyErr_Clear();
+ return name;
+}
+
+static VALUE
+rb_pygments_css(int argc, VALUE *argv, VALUE self)
+{
+ VALUE css = Qnil, prefix = Qnil, options = Qnil;
+ PyObject *args = NULL, *kwargs = NULL, *formatter = NULL;
+ int found = rb_scan_args(argc, argv, "02", &prefix, &options);
+
+ if (found == 1 && TYPE(prefix) == T_HASH) {
+ options = prefix;
+ prefix = Qnil;
+ }
+
+ if (RTEST(prefix))
+ Check_Type(prefix, T_STRING);
+
+ if (RTEST(options)) {
+ Check_Type(options, T_HASH);
+ kwargs = rb_to_py(options);
+ }
+
+ args = Py_BuildValue("(s)", "html");
+ formatter = PyObject_Call(get_formatter_by_name, args, kwargs);
+ if (formatter) {
+ PyObject *styles = PyObject_CallMethod(formatter, "get_style_defs", "(s)", RTEST(prefix) ? RSTRING_PTR(prefix) : "");
+ if (styles) {
+ css = rb_str_new2(PyString_AsString(styles));
+ }
+ Py_XDECREF(styles);
+ }
+
+ Py_XDECREF(args);
+ Py_XDECREF(kwargs);
+ Py_XDECREF(formatter);
+ PyErr_Clear();
+
+ return css;
+}
+
+static VALUE
+rb_pygments_highlight(int argc, VALUE *argv, VALUE self)
+{
+ PyObject *args = NULL, *kwargs = NULL, *lexer = NULL, *formatter = NULL;
+ VALUE code = Qnil, options = Qnil, ret = Qnil, format = Qnil;
+ rb_scan_args(argc, argv, "11", &code, &options);
+
+ if (RTEST(options)) {
+ format = rb_hash_aref(options, ID2SYM(rb_intern("formatter")));
+ if (RTEST(format))
+ Check_Type(format, T_STRING);
+
+ VALUE kw = rb_hash_aref(options, ID2SYM(rb_intern("options")));
+ if (RTEST(kw)) {
+ Check_Type(kw, T_HASH);
+ kwargs = rb_to_py(kw);
+ }
+ }
+
+ lexer = pygments__lexer_for(code, options);
+
+ if (lexer) {
+ args = Py_BuildValue("(s)", RTEST(format) ? RSTRING_PTR(format) : "html");
+ formatter = PyObject_Call(get_formatter_by_name, args, kwargs);
+ Py_XDECREF(args);
+ args = NULL;
+
+ if (formatter) {
+ PyObject *input = NULL, *output = NULL;
+
+ input = PyUnicode_FromStringAndSize(RSTRING_PTR(code), RSTRING_LEN(code));
+ if (input) {
+ output = PyObject_CallFunction(highlight, "(OOO)", input, lexer, formatter);
+
+ if (output) {
+ PyObject *string = PyUnicode_AsEncodedString(output, "utf-8", "strict");
+ if (string) {
+ Py_ssize_t len;
+ char *data;
+
+ if (PyString_AsStringAndSize(string, &data, &len) == 0) {
+ ret = rb_str_new(data, len);
+ }
+ }
+ Py_XDECREF(string);
+ }
+ }
+
+ Py_XDECREF(output);
+ Py_XDECREF(input);
+ }
+ }
+
+ Py_XDECREF(args);
+ Py_XDECREF(kwargs);
+ Py_XDECREF(lexer);
+ PyErr_Clear();
+
+#ifdef RUBY_VM
+ if (RTEST(ret))
+ rb_funcall(ret, rb_intern("force_encoding"), 1, rb_str_new2("utf-8"));
+#endif
+
+ return ret;
+}
+
+static VALUE
+rb_pygments_styles(int argc, VALUE *argv, VALUE self)
+{
+ PyObject *styles = PyObject_CallFunction(get_all_styles, "");
+ VALUE ret = py_to_rb(styles);
+ Py_XDECREF(styles);
+ PyErr_Clear();
+ return ret;
+}
+
+static VALUE
+rb_pygments_filters(int argc, VALUE *argv, VALUE self)
+{
+ PyObject *filters = PyObject_CallFunction(get_all_filters, "");
+ VALUE ret = py_to_rb(filters);
+ Py_XDECREF(filters);
+ PyErr_Clear();
+ return ret;
+}
+
+static VALUE
+rb_pygments_lexers(int argc, VALUE *argv, VALUE self)
+{
+ PyObject *lexers = PyObject_CallFunction(get_all_lexers, "");
+ VALUE ret = py_to_rb(lexers);
+ Py_XDECREF(lexers);
+ PyErr_Clear();
+ return ret;
+}
+
+static VALUE
+rb_pygments_formatters(int argc, VALUE *argv, VALUE self)
+{
+ PyObject *formatters = PyObject_CallFunction(get_all_formatters, "");
+ PyObject *item;
+ VALUE ret = rb_ary_new();
+
+ while ((item = PyIter_Next(formatters))) {
+ VALUE curr = rb_ary_new();
+ rb_ary_push(ret, curr);
+
+ rb_ary_push(curr, py_to_rb(PyObject_GetAttrString(item, "__name__")));
+ rb_ary_push(curr, py_to_rb(PyObject_GetAttrString(item, "name")));
+ rb_ary_push(curr, py_to_rb(PyObject_GetAttrString(item, "aliases")));
+
+ Py_DECREF(item);
+ }
+
+ Py_XDECREF(formatters);
+ PyErr_Clear();
+ return ret;
+}
+
+#define ENSURE(var, expr) do{ \
+ if ((var = (expr)) == NULL) { \
+ rb_raise(rb_eRuntimeError, "unable to lookup " # var); \
+ } \
+} while(0)
+
+void
+Init_pygments_ext()
+{
+ { /* python stuff */
+ Py_Initialize();
+
+ /* modules */
+ ENSURE(pygments, PyImport_ImportModule("pygments"));
+ ENSURE(pygments_lexers, PyImport_ImportModule("pygments.lexers"));
+ ENSURE(pygments_formatters, PyImport_ImportModule("pygments.formatters"));
+ ENSURE(pygments_styles, PyImport_ImportModule("pygments.styles"));
+ ENSURE(pygments_filters, PyImport_ImportModule("pygments.filters"));
+
+ /* lexer methods */
+ ENSURE(guess_lexer, PyObject_GetAttrString(pygments_lexers, "guess_lexer"));
+ ENSURE(guess_lexer_for_filename, PyObject_GetAttrString(pygments_lexers, "guess_lexer_for_filename"));
+ ENSURE(get_lexer_for_filename, PyObject_GetAttrString(pygments_lexers, "get_lexer_for_filename"));
+ ENSURE(get_lexer_for_mimetype, PyObject_GetAttrString(pygments_lexers, "get_lexer_for_mimetype"));
+ ENSURE(get_lexer_by_name, PyObject_GetAttrString(pygments_lexers, "get_lexer_by_name"));
+ ENSURE(get_all_lexers, PyObject_GetAttrString(pygments_lexers, "get_all_lexers"));
+
+ /* formatter methods */
+ ENSURE(get_formatter_by_name, PyObject_GetAttrString(pygments_formatters, "get_formatter_by_name"));
+ ENSURE(get_all_formatters, PyObject_GetAttrString(pygments_formatters, "get_all_formatters"));
+
+ /* highlighter methods */
+ ENSURE(highlight, PyObject_GetAttrString(pygments, "highlight"));
+
+ /* style methods */
+ ENSURE(get_all_styles, PyObject_GetAttrString(pygments_styles, "get_all_styles"));
+
+ /* filter methods */
+ ENSURE(get_all_filters, PyObject_GetAttrString(pygments_filters, "get_all_filters"));
+ }
+
+ { /* ruby stuff */
+ mPygments = rb_define_module("Pygments");
+ rb_define_method(mPygments, "lexer_name_for", rb_pygments_lexer_name_for, -1);
+ rb_define_method(mPygments, "css", rb_pygments_css, -1);
+ rb_define_method(mPygments, "highlight", rb_pygments_highlight, -1);
+ rb_define_method(mPygments, "styles", rb_pygments_styles, 0);
+ rb_define_method(mPygments, "filters", rb_pygments_filters, 0);
+ rb_define_method(mPygments, "_lexers", rb_pygments_lexers, 0);
+ rb_define_method(mPygments, "_formatters", rb_pygments_formatters, 0);
+ }
+}
29 lib/pygments.rb
@@ -0,0 +1,29 @@
+require 'pygments_ext'
+
+module Pygments
+ extend self
+
+ def formatters
+ _formatters.inject(Hash.new) do |hash, (name, desc, aliases)|
+ name.sub!(/Formatter$/,'')
+ hash[name] = {
+ :name => name,
+ :description => desc,
+ :aliases => aliases
+ }
+ hash
+ end
+ end
+
+ def lexers
+ _lexers.inject(Hash.new) do |hash, (name, aliases, files, mimes)|
+ hash[name] = {
+ :description => name,
+ :aliases => aliases,
+ :filenames => files,
+ :mimetypes => mimes
+ }
+ hash
+ end
+ end
+end
3  lib/pygments/version.rb
@@ -0,0 +1,3 @@
+module Pygments
+ VERSION = '0.1.0'
+end
22 pygments.rb.gemspec
@@ -0,0 +1,22 @@
+require File.expand_path('../lib/pygments/version', __FILE__)
+
+Gem::Specification.new do |s|
+ s.name = 'pygments.rb'
+ s.version = Pygments::VERSION
+
+ s.summary = 'pygments wrapper for ruby'
+ s.description = 'pygments.rb exposes the pygments syntax highlighter via embedded python'
+
+ s.homepage = 'http://github.com/tmm1/pygments.rb'
+ s.has_rdoc = false
+
+ s.authors = ['Aman Gupta']
+ s.email = ['aman@tmm1.net']
+
+ s.add_development_dependency 'rake-compiler', '0.7.6'
+
+ s.extensions = ['ext/extconf.rb']
+ s.require_paths = ['lib']
+
+ s.files = `git ls-files`.split("\n")
+end
101 test/test_pygments.rb
@@ -0,0 +1,101 @@
+# coding: utf-8
+
+require 'test/unit'
+require 'pygments'
+
+class PygmentsLexerTest < Test::Unit::TestCase
+ include Pygments
+
+ RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"
+
+ def test_lexer_by_content
+ assert_equal 'rb', lexer_name_for(RUBY_CODE)
+ end
+ def test_lexer_by_mimetype
+ assert_equal 'rb', lexer_name_for(:mimetype => 'text/x-ruby')
+ end
+ def test_lexer_by_filename
+ assert_equal 'rb', lexer_name_for(:filename => 'test.rb')
+ end
+ def test_lexer_by_name
+ assert_equal 'rb', lexer_name_for(:lexer => 'ruby')
+ end
+ def test_lexer_by_filename_and_content
+ assert_equal 'rb', lexer_name_for(RUBY_CODE, :filename => 'test.rb')
+ end
+end
+
+class PygmentsCssTest < Test::Unit::TestCase
+ include Pygments
+
+ def test_css
+ assert_match /^\.err \{/, css
+ end
+ def test_css_prefix
+ assert_match /^\.highlight \.err \{/, css('.highlight')
+ end
+ def test_css_options
+ assert_match /^\.codeerr \{/, css(:classprefix => 'code')
+ end
+ def test_css_prefix_and_options
+ assert_match /^\.mycode \.codeerr \{/, css('.mycode', :classprefix => 'code')
+ end
+ def test_css_default
+ assert_match '.c { color: #408080; font-style: italic }', css
+ end
+ def test_css_colorful
+ assert_match '.c { color: #808080 }', css(:style => 'colorful')
+ end
+end
+
+class PygmentsConfigTest < Test::Unit::TestCase
+ include Pygments
+
+ def test_styles
+ assert styles.include?('colorful')
+ end
+ def test_filters
+ assert filters.include?('codetagify')
+ end
+ def test_lexers
+ list = lexers
+ assert list.has_key?('Ruby')
+ assert list['Ruby'][:aliases].include?('duby')
+ end
+ def test_formatters
+ list = formatters
+ assert list.has_key?('Html')
+ assert list['Html'][:aliases].include?('html')
+ end
+end
+
+class PygmentsHighlightTest < Test::Unit::TestCase
+ include Pygments
+
+ RUBY_CODE = "#!/usr/bin/ruby\nputs 'foo'"
+
+ def test_highlight_defaults_to_html
+ code = highlight(RUBY_CODE)
+ assert_match '<span class="c1">#!/usr/bin/ruby</span>', code
+ end
+
+ def test_highlight_works_on_utf8
+ code = highlight('# ø', :lexer => 'rb')
+ assert_match '<span class="c1"># ø</span>', code
+ end
+
+ def test_highlight_formatter_bbcode
+ code = highlight(RUBY_CODE, :formatter => 'bbcode')
+ assert_match '[i]#!/usr/bin/ruby[/i]', code
+ end
+
+ def test_highlight_formatter_terminal
+ code = highlight(RUBY_CODE, :formatter => 'terminal')
+ assert_match "\e[37m#!/usr/bin/ruby\e[39;49;00m", code
+ end
+
+ def test_highlight_options
+ code = highlight(RUBY_CODE, :options => {:full => true, :title => 'test'})
+ assert_match '<title>test</title>', code
+ end
+end
Please sign in to comment.
Something went wrong with that request. Please try again.