Parameterize should accept malformed utf8 characters [#4323 state:resolved]

Signed-off-by: Jeremy Kemper <jeremy@bitsweat.net>

Author: Kristopher Murata

activesupport/lib/active_support/inflector/transliterate.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'iconv'
+require 'kconv'
 require 'active_support/core_ext/string/multibyte'
 
 module ActiveSupport
@@ -44,6 +45,8 @@ def transliterate(string)
     #   <%= link_to(@person.name, person_path(@person)) %>
     #   # => <a href="/person/1-donald-e-knuth">Donald E. Knuth</a>
     def parameterize(string, sep = '-')
+      # remove malformed utf8 characters
+      string = string.toutf8 unless string.is_utf8?
       # replace accented chars with their ascii equivalents
       parameterized_string = transliterate(string)
       # Turn unwanted chars into the separator
@@ -58,4 +61,4 @@ def parameterize(string, sep = '-')
       parameterized_string.downcase
     end    
   end
-end
+end

activesupport/test/inflector_test_cases.rb

@@ -159,7 +159,8 @@ module InflectorTestCases
     "Trailing bad characters!@#"          => "trailing-bad-characters",
     "!@#Leading bad characters"           => "leading-bad-characters",
     "Squeeze   separators"                => "squeeze-separators",
-    "Test with + sign"                    => "test-with-sign"
+    "Test with + sign"                    => "test-with-sign",
+    "Test with malformed utf8 \251"       => "test-with-malformed-utf8"
   }
 
   StringToParameterizeWithNoSeparator = {
@@ -168,7 +169,8 @@ module InflectorTestCases
     "Trailing bad characters!@#"          => "trailingbadcharacters",
     "!@#Leading bad characters"           => "leadingbadcharacters",
     "Squeeze   separators"                => "squeezeseparators",
-    "Test with + sign"                    => "testwithsign"
+    "Test with + sign"                    => "testwithsign",
+    "Test with malformed utf8 \251"       => "testwithmalformedutf8"
   }
 
   StringToParameterizeWithUnderscore = {
@@ -177,12 +179,14 @@ module InflectorTestCases
     "Trailing bad characters!@#"          => "trailing_bad_characters",
     "!@#Leading bad characters"           => "leading_bad_characters",
     "Squeeze   separators"                => "squeeze_separators",
-    "Test with + sign"                    => "test_with_sign"
+    "Test with + sign"                    => "test_with_sign",
+    "Test with malformed utf8 \251"       => "test_with_malformed_utf8"
   }
 
   StringToParameterizedAndNormalized = {
     "Malmö"                               => "malmo",
-    "Garçons"                             => "garcons"
+    "Garçons"                             => "garcons",
+    "Ops\331"                            => "ops"
   }
 
   UnderscoreToHuman = {