Skip to content
Browse files

Add ActiveSupport::Multibyte::Chars#limit.

The limit method limits the number of bytes in a string. Useful when the
storage space of the string is limited, for instance in a database column
definition.

Sharpen up the implementation of translate offset.

[#3192 state:committed]
  • Loading branch information...
1 parent a3d5274 commit 935bd0fef8e26f4ec65fe411a1d29942493f8d46 @Manfred Manfred committed Nov 1, 2009
Showing with 74 additions and 14 deletions.
  1. +15 −14 activesupport/lib/active_support/multibyte/chars.rb
  2. +59 −0 activesupport/test/multibyte_chars_test.rb
View
29 activesupport/lib/active_support/multibyte/chars.rb
@@ -363,6 +363,16 @@ def slice!(*args)
slice
end
+ # Limit the byte size of the string to a number of bytes without breaking characters. Usable
+ # when the storage for a string is limited for some reason.
+ #
+ # Example:
+ # s = 'こんにちは'
+ # s.mb_chars.limit(7) #=> "こに"
+ def limit(limit)
+ slice(0...translate_offset(limit))
+ end
+
# Returns the codepoint of the first character in the string.
#
# Example:
@@ -651,24 +661,15 @@ def tidy_bytes(string)
end
protected
-
+
def translate_offset(byte_offset) #:nodoc:
return nil if byte_offset.nil?
return 0 if @wrapped_string == ''
- chunk = @wrapped_string[0..byte_offset]
begin
- begin
- chunk.unpack('U*').length - 1
- rescue ArgumentError => e
- chunk = @wrapped_string[0..(byte_offset+=1)]
- # Stop retrying at the end of the string
- raise e unless byte_offset < chunk.length
- # We damaged a character, retry
- retry
- end
- # Catch the ArgumentError so we can throw our own
- rescue ArgumentError
- raise EncodingError, 'malformed UTF-8 character'
+ @wrapped_string[0...byte_offset].unpack('U*').length
+ rescue ArgumentError => e
+ byte_offset -= 1
+ retry
end
end
View
59 activesupport/test/multibyte_chars_test.rb
@@ -169,6 +169,7 @@ def test_string_methods_are_chainable
assert chars('').strip.kind_of?(ActiveSupport::Multibyte.proxy_class)
assert chars('').reverse.kind_of?(ActiveSupport::Multibyte.proxy_class)
assert chars(' ').slice(0).kind_of?(ActiveSupport::Multibyte.proxy_class)
+ assert chars('').limit(0).kind_of?(ActiveSupport::Multibyte.proxy_class)
assert chars('').upcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
assert chars('').downcase.kind_of?(ActiveSupport::Multibyte.proxy_class)
assert chars('').capitalize.kind_of?(ActiveSupport::Multibyte.proxy_class)
@@ -196,7 +197,9 @@ def test_sortability
def test_should_return_character_offset_for_regexp_matches
assert_nil(@chars =~ /wrong/u)
assert_equal 0, (@chars =~ //u)
+ assert_equal 0, (@chars =~ /こに/u)
assert_equal 1, (@chars =~ //u)
+ assert_equal 2, (@chars =~ //u)
assert_equal 3, (@chars =~ //u)
end
@@ -493,6 +496,44 @@ def test_capitalize_should_be_unicode_aware
end
end
+ def test_limit_should_not_break_on_blank_strings
+ chars = ''.mb_chars
+
+ assert_equal '', chars.limit(0)
+ assert_equal '', chars.limit(1)
+ end
+
+ def test_limit_should_work_on_a_multibyte_string
+ chars = UNICODE_STRING.mb_chars
+
+ assert_equal UNICODE_STRING, chars.limit(UNICODE_STRING.length)
+ assert_equal '', chars.limit(0)
+ assert_equal '', chars.limit(1)
+ assert_equal '', chars.limit(3)
+ assert_equal 'こに', chars.limit(6)
+ assert_equal 'こに', chars.limit(8)
+ assert_equal 'こにち', chars.limit(9)
+ assert_equal 'こにちわ', chars.limit(50)
+ end
+
+ def test_limit_should_work_on_an_ascii_string
+ ascii = ASCII_STRING.mb_chars
+
+ assert_equal ASCII_STRING, ascii.limit(ASCII_STRING.length)
+ assert_equal '', ascii.limit(0)
+ assert_equal 'o', ascii.limit(1)
+ assert_equal 'oh', ascii.limit(2)
+ assert_equal 'ohay', ascii.limit(4)
+ assert_equal 'ohayo', ascii.limit(50)
+ end
+
+ def test_limit_should_keep_under_the_specified_byte_limit
+ chars = UNICODE_STRING.mb_chars
+ (1..UNICODE_STRING.length).each do |limit|
+ assert chars.limit(limit).to_s.length <= limit
+ end
+ end
+
def test_composition_exclusion_is_set_up_properly
# Normalization of DEVANAGARI LETTER QA breaks when composition exclusion isn't used correctly
qa = [0x915, 0x93c].pack('U*')
@@ -603,3 +644,21 @@ def string_from_classes(classes)
end.pack('U*')
end
end
+
+class MultibyteInternalsTest < ActiveSupport::TestCase
+ include MultibyteTestHelpers
+
+ test "Chars translates a character offset to a byte offset" do
+ chars = "Puisque c'était son erreur, il m'a aidé".mb_chars
+ [
+ [0, 0],
+ [3, 3],
+ [12, 11],
+ [14, 13],
+ [41, 39]
+ ].each do |byte_offset, character_offset|
+ assert_equal character_offset, chars.send(:translate_offset, byte_offset),
+ "Expected byte offset #{byte_offset} to translate to #{character_offset}"
+ end
+ end
+end

0 comments on commit 935bd0f

Please sign in to comment.
Something went wrong with that request. Please try again.