Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Use detect language

  • Loading branch information...
commit b0414e232aad9b4e21d1cb1393f211f0eda25711 1 parent 2a5865a
@josh josh authored
View
2  README.md
@@ -358,7 +358,7 @@ submit a pull request.
**3.0.0**
-* MIME Types now accept charset custom charset decoders. Improves support for UTF-16/32 files.
+* MIME Types now accept charset custom charset detecters. Improves support for UTF-16/32 files.
* Environment#version no longer affects asset digests. Only used for busting the asset cache.
* Removed builtin support for LESS.
* Removed include directive support.
View
10 lib/sprockets.rb
@@ -55,13 +55,13 @@ module Sprockets
@compressors = Hash.new { |h, k| h[k] = {} }
# Common asset text types
- register_mime_type 'application/javascript', type: :text, extensions: ['.js'], decoder: EncodingUtils.method(:decode_unicode)
- register_mime_type 'application/json', type: :text, extensions: ['.json'], decoder: EncodingUtils.method(:decode_unicode)
- register_mime_type 'application/x-ruby', type: :text, extensions: ['.rb'], decoder: EncodingUtils.method(:decode_unicode)
- register_mime_type 'text/css', type: :text, extensions: ['.css'], decoder: EncodingUtils.method(:decode_css)
+ register_mime_type 'application/javascript', type: :text, extensions: ['.js'], charset: EncodingUtils::DETECT_UNICODE
+ register_mime_type 'application/json', type: :text, extensions: ['.json'], charset: EncodingUtils::DETECT_UNICODE
+ register_mime_type 'application/x-ruby', type: :text, extensions: ['.rb'], charset: EncodingUtils::DETECT_UNICODE
+ register_mime_type 'text/css', type: :text, extensions: ['.css'], charset: EncodingUtils::DETECT_CSS
register_mime_type 'text/html', type: :text, extensions: ['.html', '.htm']
register_mime_type 'text/plain', type: :text, extensions: ['.txt', '.text']
- register_mime_type 'text/yaml', type: :text, extensions: ['.yml', '.yaml'], decoder: EncodingUtils.method(:decode_unicode)
+ register_mime_type 'text/yaml', type: :text, extensions: ['.yml', '.yaml'], charset: EncodingUtils::DETECT_UNICODE
# Common asset binary types
register_mime_type 'application/vnd.ms-fontobject', type: :binary, extensions: ['.eot']
View
2  lib/sprockets/base.rb
@@ -261,7 +261,7 @@ def build_processed_asset_hash(asset, processors)
data = File.open(filename, 'rb') { |f| f.read }
if type = mime_types[asset[:content_type]]
- data = type[:decoder].call(data) if type[:decoder]
+ data = type[:charset].call(data) if type[:charset]
data = data.encode(Encoding::UTF_8) if type[:type] == :text
end
View
31 lib/sprockets/encoding_utils.rb
@@ -11,7 +11,7 @@ module EncodingUtils
Encoding::UTF_16BE => [0xFE, 0xFF]
}
- # Public: Basic string decoder.
+ # Public: Basic string detecter.
#
# Attempts to parse any Unicode BOM otherwise falls back to the
# environment's external encoding.
@@ -19,8 +19,8 @@ module EncodingUtils
# str - ASCII-8BIT encoded String
#
# Returns encoded String.
- def decode(str)
- str = decode_unicode_bom(str)
+ def detect(str)
+ str = detect_unicode_bom(str)
# Fallback to UTF-8
if str.encoding == Encoding::BINARY
@@ -30,15 +30,18 @@ def decode(str)
str
end
- # Public: Decode Unicode string.
+ # Public: Alias for EncodingUtils.detect_unicode
+ DETECT = method(:detect)
+
+ # Public: Detect Unicode string.
#
# Attempts to parse Unicode BOM and falls back to UTF-8.
#
# str - ASCII-8BIT encoded String
#
# Returns encoded String.
- def decode_unicode(str)
- str = decode_unicode_bom(str)
+ def detect_unicode(str)
+ str = detect_unicode_bom(str)
# Fallback to UTF-8
if str.encoding == Encoding::BINARY
@@ -48,13 +51,16 @@ def decode_unicode(str)
str
end
- # Public: Decode and strip BOM from possible unicode string.
+ # Public: Alias for EncodingUtils.detect_unicode
+ DETECT_UNICODE = method(:detect_unicode)
+
+ # Public: Detect and strip BOM from possible unicode string.
#
# str - ASCII-8BIT encoded String
#
# Returns UTF 8/16/32 encoded String without BOM or the original String if
# no BOM was present.
- def decode_unicode_bom(str)
+ def detect_unicode_bom(str)
bom_bytes = str.byteslice(0, 4).bytes.to_a
BOM.each do |encoding, bytes|
@@ -70,13 +76,13 @@ def decode_unicode_bom(str)
return str
end
- # Public: Decode and strip @charset from CSS style sheet.
+ # Public: Detect and strip @charset from CSS style sheet.
#
# str - String.
#
# Returns a encoded String.
- def decode_css(str)
- str = decode_unicode_bom(str)
+ def detect_css(str)
+ str = detect_unicode_bom(str)
if name = scan_css_charset(str)
encoding = Encoding.find(name)
@@ -95,6 +101,9 @@ def decode_css(str)
str
end
+ # Public: Alias for EncodingUtils.detect_css
+ DETECT_CSS = method(:detect_css)
+
# Internal: @charset bytes
CHARSET_START = [0x40, 0x63, 0x68, 0x61, 0x72, 0x73, 0x65, 0x74, 0x20, 0x22]
View
8 lib/sprockets/mime.rb
@@ -8,7 +8,7 @@ module Mime
# value - Hash
# type - :text or :binary
# extensions - Array of extnames
- # decoder - Proc to decode binary content
+ # charset - Default Encoding or function to detect encoding
#
# Returns Hash.
attr_reader :mime_types
@@ -31,8 +31,8 @@ def register_mime_type(mime_type, options = {})
raise ArgumentError, "type must be :binary or :text"
end
- decoder = options[:decoder]
- decoder ||= EncodingUtils.method(:decode) if type == :text
+ charset = options[:charset]
+ charset ||= EncodingUtils::DETECT if type == :text
extnames.each do |extname|
@mime_exts[extname] = mime_type
@@ -41,7 +41,7 @@ def register_mime_type(mime_type, options = {})
@mime_types[mime_type] = {}
@mime_types[mime_type][:type] = type
@mime_types[mime_type][:extensions] = extnames
- @mime_types[mime_type][:decoder] = decoder if decoder
+ @mime_types[mime_type][:charset] = charset if charset
@mime_types[mime_type]
end
View
30 test/test_encoding.rb
@@ -4,76 +4,76 @@
class EncodingUtilsTest < Sprockets::TestCase
include Sprockets::EncodingUtils
- test "decode unicode bom" do
+ test "detect unicode bom" do
str = File.open(fixture_path('encoding/ascii.js'), 'rb') { |f| f.read }
assert_equal Encoding::BINARY, str.encoding
assert_equal 17, str.bytesize
- str = decode_unicode_bom(str)
+ str = detect_unicode_bom(str)
assert_equal Encoding::BINARY, str.encoding
assert_equal 17, str.bytesize
str = File.open(fixture_path('encoding/utf8.js'), 'rb') { |f| f.read }
assert_equal Encoding::BINARY, str.encoding
assert_equal 20, str.bytesize
- str = decode_unicode_bom(str)
+ str = detect_unicode_bom(str)
assert_equal Encoding::BINARY, str.encoding
assert_equal 20, str.bytesize
str = File.open(fixture_path('encoding/utf8_bom.js'), 'rb') { |f| f.read }
assert_equal Encoding::BINARY, str.encoding
assert_equal 23, str.bytesize
- str = decode_unicode_bom(str)
+ str = detect_unicode_bom(str)
assert_equal Encoding::UTF_8, str.encoding
assert_equal 20, str.bytesize
str = File.open(fixture_path('encoding/utf8_bom.js'), 'rb') { |f| f.read }
str.force_encoding(Encoding::UTF_8)
assert_equal 23, str.bytesize
- str = decode_unicode_bom(str)
+ str = detect_unicode_bom(str)
assert_equal Encoding::UTF_8, str.encoding
assert_equal 20, str.bytesize
str = File.open(fixture_path('encoding/utf16le.js'), 'rb') { |f| f.read }
assert_equal Encoding::BINARY, str.encoding
assert_equal 38, str.bytesize
- str = decode_unicode_bom(str)
+ str = detect_unicode_bom(str)
assert_equal Encoding::UTF_16LE, str.encoding
assert_equal 36, str.bytesize
str = File.open(fixture_path('encoding/utf16le.js'), 'rb') { |f| f.read }
str.force_encoding(Encoding::UTF_16LE)
assert_equal 38, str.bytesize
- str = decode_unicode_bom(str)
+ str = detect_unicode_bom(str)
assert_equal Encoding::UTF_16LE, str.encoding
assert_equal 36, str.bytesize
str = File.open(fixture_path('encoding/utf16be.js'), 'rb') { |f| f.read }
assert_equal Encoding::BINARY, str.encoding
assert_equal 38, str.bytesize
- str = decode_unicode_bom(str)
+ str = detect_unicode_bom(str)
assert_equal Encoding::UTF_16BE, str.encoding
assert_equal 36, str.bytesize
str = File.open(fixture_path('encoding/utf32le.js'), 'rb') { |f| f.read }
assert_equal Encoding::BINARY, str.encoding
assert_equal 76, str.bytesize
- str = decode_unicode_bom(str)
+ str = detect_unicode_bom(str)
assert_equal Encoding::UTF_32LE, str.encoding
assert_equal 72, str.bytesize
str = File.open(fixture_path('encoding/utf32be.js'), 'rb') { |f| f.read }
assert_equal Encoding::BINARY, str.encoding
assert_equal 76, str.bytesize
- str = decode_unicode_bom(str)
+ str = detect_unicode_bom(str)
assert_equal Encoding::UTF_32BE, str.encoding
assert_equal 72, str.bytesize
end
- test "decode css charset" do
+ test "detect css charset" do
str = File.open(fixture_path('encoding/utf8-charset.css'), 'rb') { |f| f.read }
assert_equal Encoding::BINARY, str.encoding
assert_equal 38, str.bytesize
- str = decode_css(str)
+ str = detect_css(str)
assert_equal Encoding::UTF_8, str.encoding
assert_equal 21, str.bytesize
assert_equal "\n\nh1 { color: red; }\n", str.encode(Encoding::UTF_8)
@@ -81,7 +81,7 @@ class EncodingUtilsTest < Sprockets::TestCase
str = File.open(fixture_path('encoding/utf8-charset.css'), 'rb') { |f| f.read }
str.force_encoding(Encoding::UTF_8)
assert_equal 38, str.bytesize
- str = decode_css(str)
+ str = detect_css(str)
assert_equal Encoding::UTF_8, str.encoding
assert_equal 21, str.bytesize
assert_equal "\n\nh1 { color: red; }\n", str.encode(Encoding::UTF_8)
@@ -89,7 +89,7 @@ class EncodingUtilsTest < Sprockets::TestCase
str = File.open(fixture_path('encoding/utf16le-charset.css'), 'rb') { |f| f.read }
assert_equal Encoding::BINARY, str.encoding
assert_equal 82, str.bytesize
- str = decode_css(str)
+ str = detect_css(str)
assert_equal Encoding::UTF_16LE, str.encoding
assert_equal 42, str.bytesize
assert_equal "\n\nh1 { color: red; }\n", str.encode(Encoding::UTF_8)
@@ -97,7 +97,7 @@ class EncodingUtilsTest < Sprockets::TestCase
str = File.open(fixture_path('encoding/utf16le-charset.css'), 'rb') { |f| f.read }
str.force_encoding(Encoding::UTF_16LE)
assert_equal 82, str.bytesize
- str = decode_css(str)
+ str = detect_css(str)
assert_equal Encoding::UTF_16LE, str.encoding
assert_equal 42, str.bytesize
assert_equal "\n\nh1 { color: red; }\n", str.encode(Encoding::UTF_8)
Please sign in to comment.
Something went wrong with that request. Please try again.