Skip to content
Browse files

Replace Lexer with Pygments::Lexer

  • Loading branch information...
1 parent bb11317 commit 0864b9847f38a8972b18276476c9f74f237320b7 @josh josh committed
View
2 Gemfile
@@ -1,2 +1,4 @@
source :rubygems
gemspec
+
+gem 'pygments.rb', :git => 'git://github.com/tmm1/pygments.rb.git'
View
14 Rakefile
@@ -1,4 +1,3 @@
-require 'rake/clean'
require 'rake/testtask'
task :default => :test
@@ -6,16 +5,3 @@ task :default => :test
Rake::TestTask.new do |t|
t.warning = true
end
-
-CLOBBER.include 'lib/linguist/lexers.yml'
-
-file 'lib/linguist/lexers.yml' do |f|
- # GitHub vendored pygments path
- # vendor/python/pygments
- path = File.expand_path('../../../python/pygments', __FILE__)
- ENV['PYTHONPATH'] = path if File.directory?(path)
-
- sh "python ./bin/pygments-lexers > #{f.name}"
-end
-
-task :lexers => 'lib/linguist/lexers.yml'
View
27 bin/pygments-lexers
@@ -1,27 +0,0 @@
-#!/usr/bin/env python
-
-from pygments import __version__
-from pygments.lexers import get_all_lexers
-
-print "# AUTOMATICALLY GENERATED BY bin/pygments-lexers"
-print "# DO NOT MODIFIY. YOUR PULL WILL BE REJECTED."
-print "# Pygments version %s" % __version__
-print "---"
-
-for name, aliases, filenames, mimetypes in get_all_lexers():
- print "%s:" % name
-
- if any(aliases):
- print " aliases:"
- for alias in aliases:
- print " - '%s'" % alias
-
- if any(filenames):
- print " filenames:"
- for filename in filenames:
- print " - '%s'" % filename
-
- if any(mimetypes):
- print " mimetypes:"
- for mimetype in mimetypes:
- print " - '%s'" % mimetype
View
19 lib/linguist/blob_helper.rb
@@ -3,6 +3,7 @@
require 'linguist/pathname'
require 'escape_utils'
+require 'pygments'
require 'yaml'
module Linguist
@@ -343,7 +344,7 @@ def guess_language
#
# Returns a Lexer.
def lexer
- language ? language.lexer : Lexer['Text only']
+ language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
end
# Internal: Disambiguates between multiple language extensions.
@@ -512,19 +513,27 @@ def shebang_language
# Public: Highlight syntax of blob
#
+ # options - A Hash of options (defaults to {})
+ #
# Returns html String
- def colorize
+ def colorize(options = {})
return if !text? || large?
- lexer.colorize(data)
+ lexer.highlight(data, options)
end
# Public: Highlight syntax of blob without the outer highlight div
# wrapper.
#
+ # options - A Hash of options (defaults to {})
+ #
# Returns html String
- def colorize_without_wrapper
+ def colorize_without_wrapper(options = {})
return if !text? || large?
- lexer.colorize_without_wrapper(data)
+ if text = lexer.highlight(data, options)
+ text[%r{<div class="highlight"><pre>(.*?)</pre>\s*</div>}m, 1]
+ else
+ ''
+ end
end
Language.overridden_extensions.each do |extension|
View
22 lib/linguist/language.rb
@@ -1,6 +1,5 @@
-require 'linguist/lexer'
-
require 'yaml'
+require 'pygments'
module Linguist
# Language names that are recognizable by GitHub. Defined languages
@@ -211,7 +210,7 @@ def initialize(attributes = {})
@aliases = [default_alias_name] + (attributes[:aliases] || [])
# Lookup Lexer object
- @lexer = Lexer.find_by_name(attributes[:lexer] || name) ||
+ @lexer = Pygments::Lexer.find_by_name(attributes[:lexer] || name) ||
raise(ArgumentError, "#{@name} is missing lexer")
# Set legacy search term
@@ -370,21 +369,12 @@ def searchable?
# Public: Highlight syntax of text
#
- # text - String of code to be highlighted
- #
- # Returns html String
- def colorize(text)
- lexer.colorize(text)
- end
-
- # Public: Highlight syntax of text without the outer highlight div
- # wrapper.
- #
- # text - String of code to be highlighted
+ # text - String of code to be highlighted
+ # options - A Hash of options (defaults to {})
#
# Returns html String
- def colorize_without_wrapper(text)
- lexer.colorize_without_wrapper(text)
+ def colorize(text, options = {})
+ lexer.highlight(text, options = {})
end
# Public: Return name as String representation
View
167 lib/linguist/lexer.rb
@@ -1,167 +0,0 @@
-require 'pygments'
-require 'yaml'
-
-module Linguist
- # Mirror of Pygments Lexer structure.
- #
- # name - Proper lexer name (JavaScript, Ruby, Python)
- # aliases - Aliases for lookup (js, javascript)
- # filenames - Filename globs (*.js)
- # mimetypes - Mime types (application/javascript)
- class Lexer < Struct.new(:name, :aliases, :filenames, :mimetypes)
- @lexers = []
- @index = {}
- @name_index = {}
- @alias_index = {}
- @mimetypes_index = {}
-
- # Internal: Create a new Lexer object
- #
- # name - Name of Lexer
- # attrs - A hash of attributes
- #
- # Returns a Lexer object
- def self.create(name, attrs)
- name = name
- aliases = attrs['aliases'] || []
- filenames = attrs['filenames'] || []
- mimetypes = attrs['mimetypes'] || []
-
- @lexers << lexer = new(name, aliases, filenames, mimetypes)
-
- # All Lexer names should be unique. Warn if there is a duplicate.
- if @name_index.key?(lexer.name)
- warn "Duplicate lexer name: #{lexer.name}"
- end
-
- @index[lexer.name] = @name_index[lexer.name] = lexer
-
- lexer.aliases.each do |name|
- # All Lexer aliases should be unique. Warn if there is a duplicate.
- if @alias_index.key?(name)
- warn "Duplicate alias: #{name}"
- end
-
- @index[name] = @alias_index[name] = lexer
- end
-
- lexer.mimetypes.each do |type|
- # All Lexer mimetypes should be unique. Warn if there is a duplicate.
- if @mimetypes_index.key?(name)
- warn "Duplicate mimetype: #{name}"
- end
-
- @mimetypes_index[type] = lexer
- end
- end
-
- # Public: Get all Lexers
- #
- # Returns an Array of Lexers
- def self.all
- @lexers
- end
-
- # Public: Look up Lexer by name or alias.
- #
- # name - A String name or alias
- #
- # Lexer['Ruby']
- # => #<Lexer name="Ruby">
- #
- # Returns the Lexer or nil if none was found.
- def self.[](name)
- @index[name]
- end
-
- # Public: Look up Lexer by its proper name.
- #
- # name - The String name of the Lexer
- #
- # Examples
- #
- # Lexer.find_by_name('Ruby')
- # # => #<Lexer name="Ruby">
- #
- # Returns the Lexer or nil if none was found.
- def self.find_by_name(name)
- @name_index[name]
- end
-
- # Public: Look up Lexer by one of its aliases.
- #
- # name - A String alias of the Lexer
- #
- # Examples
- #
- # Lexer.find_by_alias('rb')
- # # => #<Lexer name="Ruby">
- #
- # Returns the Lexer or nil if none was found.
- def self.find_by_alias(name)
- @alias_index[name]
- end
-
- # Public: Look up Lexer by one of it's mime types.
- #
- # type - A mime type String.
- #
- # Examples
- #
- # Lexer.find_by_mimetype('application/x-ruby')
- # # => #<Lexer name="Ruby">
- #
- # Returns the Lexer or nil if none was found.
- def self.find_by_mimetype(type)
- @mimetypes_index[type]
- end
-
- # Public: Return a alias of the Lexer to pass to Pygments.
- #
- # The alias we choose is arbitrary.
- #
- # Returns the alias String
- def to_s
- aliases.first
- end
-
- # Public: Highlight syntax of text
- #
- # text - String of code to be highlighted
- #
- # Returns html String
- def colorize(text)
- Pygments.highlight(text, :lexer => aliases.first, :options => {:stripnl => false})
- end
-
- # Public: Highlight syntax of text without the outer highlight div
- # wrapper.
- #
- # text - String of code to be highlighted
- #
- # Returns html String
- def colorize_without_wrapper(text)
- if text = colorize(text)
- text[%r{<div class="highlight"><pre>(.*?)</pre>\s*</div>}m, 1]
- else
- ''
- end
- end
-
- def ==(other)
- eql?(other)
- end
-
- def eql?(other)
- equal?(other)
- end
-
- # Load lexers from lexers.yml
- #
- # `bin/pygments-lexers` dumps a YAML list of all the available
- # Pygments lexers.
- YAML.load_file(File.expand_path("../lexers.yml", __FILE__)).each do |name, attrs|
- Lexer.create(name, attrs)
- end
- end
-end
View
1,652 lib/linguist/lexers.yml
0 additions, 1,652 deletions not shown because the diff is too large. Please use a local Git client to view these changes.
View
3 lib/linguist/pathname.rb
@@ -1,5 +1,6 @@
require 'linguist/language'
require 'linguist/mime'
+require 'pygments'
module Linguist
# Similar to ::Pathname, Linguist::Pathname wraps a path string and
@@ -61,7 +62,7 @@ def language
#
# Returns a Lexer.
def lexer
- language ? language.lexer : Lexer['Text only']
+ language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
end
# Public: Get the mime type
View
3 test/test_blob.rb
@@ -2,10 +2,13 @@
require 'test/unit'
require 'mime/types'
+require 'pygments'
class TestBlob < Test::Unit::TestCase
include Linguist
+ Lexer = Pygments::Lexer
+
def fixtures_path
File.expand_path("../fixtures", __FILE__)
end
View
25 test/test_language.rb
@@ -1,10 +1,13 @@
require 'linguist/language'
require 'test/unit'
+require 'pygments'
class TestLanguage < Test::Unit::TestCase
include Linguist
+ Lexer = Pygments::Lexer
+
def test_ambiguous_extensions
assert Language.ambiguous?('.h')
assert_equal Language['C'], Language.find_by_extension('h')
@@ -311,26 +314,4 @@ def test_colorize
</div>
HTML
end
-
- def test_colorize_without_wrapper
- assert_equal <<-HTML, Language['Text'].colorize_without_wrapper("Hello")
-Hello
- HTML
-
- assert_equal <<-HTML, Language['Ruby'].colorize_without_wrapper("def foo\n 'foo'\nend\n")
-<span class="k">def</span> <span class="nf">foo</span>
- <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
- HTML
- end
-
- def test_colorize_doesnt_strip_newlines
- assert_equal <<-HTML, Language['Ruby'].colorize_without_wrapper("\n\n# Foo\ndef 'foo'\nend\n")
-
-
-<span class="c1"># Foo</span>
-<span class="k">def</span> <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
- HTML
- end
end
View
91 test/test_lexer.rb
@@ -1,91 +0,0 @@
-require 'linguist/lexer'
-
-require 'test/unit'
-
-class TestLexer < Test::Unit::TestCase
- include Linguist
-
- def test_find_by_name
- assert_equal Lexer['Ruby'], Lexer.find_by_name('Ruby')
- end
-
- def test_find_all_by_name
- Lexer.all.each do |lexer|
- assert_equal lexer, Lexer.find_by_name(lexer.name)
- assert_equal lexer, Lexer[lexer.name]
- end
- end
-
- def test_find_by_alias
- assert_equal Lexer['Ruby'], Lexer.find_by_alias('rb')
- assert_equal Lexer['Ruby'], Lexer.find_by_alias('ruby')
- end
-
- def test_find_all_by_alias
- Lexer.all.each do |lexer|
- lexer.aliases.each do |name|
- assert_equal lexer, Lexer.find_by_alias(name)
- assert_equal lexer, Lexer[name]
- end
- end
- end
-
- def test_find_by_mimetype
- assert_equal Lexer['Ruby'], Lexer.find_by_mimetype('text/x-ruby')
- assert_equal Lexer['Ruby'], Lexer.find_by_mimetype('application/x-ruby')
- end
-
- def test_find_all_by_mimetype
- Lexer.all.each do |lexer|
- lexer.mimetypes.each do |type|
- assert_equal lexer, Lexer.find_by_mimetype(type)
- end
- end
- end
-
- def test_name
- assert_equal 'Ruby', Lexer['Ruby'].name
- assert_equal 'Python', Lexer['Python'].name
- assert_equal 'Perl', Lexer['Perl'].name
- end
-
- def test_aliases
- assert_equal ['rb', 'ruby', 'duby'], Lexer['Ruby'].aliases
- assert_equal ['python', 'py'], Lexer['Python'].aliases
- assert_equal ['perl', 'pl'], Lexer['Perl'].aliases
- end
-
- def test_eql
- assert Lexer['Ruby'].eql?(Lexer['Ruby'])
- assert !Lexer['Ruby'].eql?(Lexer['Python'])
- assert !Lexer['Ruby'].eql?(Lexer.new('Ruby'))
- end
-
- def test_colorize
- assert_equal <<-HTML, Lexer['Text only'].colorize("Hello")
-<div class="highlight"><pre>Hello
-</pre>
-</div>
- HTML
-
- assert_equal <<-HTML, Lexer['Ruby'].colorize("def foo\n 'foo'\nend\n")
-<div class="highlight"><pre><span class="k">def</span> <span class="nf">foo</span>
- <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
-</pre>
-</div>
- HTML
- end
-
- def test_colorize_without_wrapper
- assert_equal <<-HTML, Lexer['Text only'].colorize_without_wrapper("Hello")
-Hello
- HTML
-
- assert_equal <<-HTML, Lexer['Ruby'].colorize_without_wrapper("def foo\n 'foo'\nend\n")
-<span class="k">def</span> <span class="nf">foo</span>
- <span class="s1">&#39;foo&#39;</span>
-<span class="k">end</span>
- HTML
- end
-end
View
3 test/test_pathname.rb
@@ -1,10 +1,13 @@
require 'linguist/pathname'
require 'test/unit'
+require 'pygments'
class TestPathname < Test::Unit::TestCase
include Linguist
+ Lexer = Pygments::Lexer
+
def test_to_s
assert_equal "file.rb", Pathname.new("file.rb").to_s
end

0 comments on commit 0864b98

Please sign in to comment.
Something went wrong with that request. Please try again.