Decimal class (#629)

* Support decimal_class option

* Allow bigdecimal_load in default on JSON.parse

* Fix invalid encoding detection issue

* Remove string from encoding error

* Set version

CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+## 3.10.17 - 2020-12-15
+
+- The undocumented JSON gem option of `:decimal_class` is now
+  supported and the default option of `:bigdecimal_load` is also
+  honored in JSON.parse() and in compat mode.
+
+- Invalid encoding detection bug fixed for rails.
+
 ## 3.10.16 - 2020-11-10
 
 - Allow escaping any character in :compat mode to match the json gem behavior.

ext/oj/dump.c

@@ -174,15 +174,20 @@ hixss_friendly_size(const uint8_t *str, size_t len) {
     return size - len * (size_t)'0' + check;
 }
 
-inline static size_t
+inline static long
 rails_xss_friendly_size(const uint8_t *str, size_t len) {
-    size_t	size = 0;
+    long	size = 0;
     size_t	i = len;
+    uint8_t	hi = 0;
 
     for (; 0 < i; str++, i--) {
 	size += rails_xss_friendly_chars[*str];
+	hi |= *str & 0x80;
     }
-    return size - len * (size_t)'0';
+    if (0 == hi) {
+	return size - len * (size_t)'0';
+    }
+    return -(size - len * (size_t)'0');
 }
 
 inline static size_t
@@ -249,7 +254,6 @@ dump_hex(uint8_t c, Out out) {
 
 static void
 raise_invalid_unicode(const char *str, int len, int pos) {
-    char	buf[len + 1];
     char	c;
     char	code[32];
     char	*cp = code;
@@ -268,8 +272,7 @@ raise_invalid_unicode(const char *str, int len, int pos) {
     cp--;
     *cp++ = ']';
     *cp = '\0';
-    strncpy(buf, str, len);
-    rb_raise(oj_json_generator_error_class, "Invalid Unicode %s at %d in '%s'", code, pos, buf);
+    rb_raise(oj_json_generator_error_class, "Invalid Unicode %s at %d", code, pos);
 }
 
 static const char*
@@ -767,6 +770,7 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
     size_t	size;
     char	*cmap;
     const char	*orig = str;
+    bool	has_hi = false;
 
     switch (out->opts->escape_mode) {
     case NLEsc:
@@ -785,10 +789,19 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 	cmap = hixss_friendly_chars;
 	size = hixss_friendly_size((uint8_t*)str, cnt);
 	break;
-    case RailsXEsc:
+    case RailsXEsc: {
+	long	sz;
+
 	cmap = rails_xss_friendly_chars;
-	size = rails_xss_friendly_size((uint8_t*)str, cnt);
+	sz = rails_xss_friendly_size((uint8_t*)str, cnt);
+	if (sz < 0) {
+	    has_hi = true;
+	    size = (size_t)-sz;
+	} else {
+	    size = (size_t)sz;
+	}
 	break;
+    }
     case RailsEsc:
 	cmap = rails_friendly_chars;
 	size = rails_friendly_size((uint8_t*)str, cnt);
@@ -812,7 +825,7 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) {
 	str++;
 	is_sym = 0; // just to make sure
     }
-    if (cnt == size) {
+    if (cnt == size && !has_hi) {
 	if (is_sym) {
 	    *out->cur++ = ':';
 	}

ext/oj/mimic_json.c

@@ -510,8 +510,6 @@ mimic_parse_core(int argc, VALUE *argv, VALUE self, bool bang) {
     pi.options.create_ok = No;
     pi.options.allow_nan = (bang ? Yes : No);
     pi.options.nilnil = No;
-    //pi.options.bigdec_load = FloatDec;
-    pi.options.bigdec_load = RubyDec;
     pi.options.mode = CompatMode;
     pi.max_depth = 100;
 
@@ -561,6 +559,16 @@ mimic_parse_core(int argc, VALUE *argv, VALUE self, bool bang) {
 		pi.options.array_class = v;
 	    }
 	}
+	if (Qtrue == rb_funcall(ropts, oj_has_key_id, 1, oj_decimal_class_sym)) {
+	    v = rb_hash_lookup(ropts, oj_decimal_class_sym);
+	    if (rb_cFloat == v) {
+		pi.options.bigdec_load = FloatDec;
+	    } else if (oj_bigdecimal_class == v) {
+		pi.options.bigdec_load = BigDec;
+	    } else if (Qnil == v) {
+		pi.options.bigdec_load = AutoDec;
+	    }
+	}
 	v = rb_hash_lookup(ropts, oj_max_nesting_sym);
 	if (Qtrue == v) {
 	    pi.max_depth = 100;

ext/oj/oj.c

@@ -88,6 +88,7 @@ VALUE	oj_slash_string;
 VALUE	oj_allow_nan_sym;
 VALUE	oj_array_class_sym;
 VALUE	oj_create_additions_sym;
+VALUE	oj_decimal_class_sym;
 VALUE	oj_hash_class_sym;
 VALUE	oj_indent_sym;
 VALUE	oj_object_class_sym;
@@ -581,6 +582,19 @@ oj_parse_options(VALUE ropts, Options copts) {
 	    rb_raise(rb_eArgError, ":bigdecimal_load must be :bigdecimal, :float, or :auto.");
 	}
     }
+    if (Qtrue == rb_funcall(ropts, oj_has_key_id, 1, oj_decimal_class_sym)) {
+	v = rb_hash_lookup(ropts, oj_decimal_class_sym);
+	if (rb_cFloat == v) {
+	    copts->bigdec_load = FloatDec;
+	} else if (oj_bigdecimal_class == v) {
+ 	    copts->bigdec_load = BigDec;
+	} else if (Qnil == v) {
+	    copts->bigdec_load = AutoDec;
+	} else {
+	    rb_raise(rb_eArgError, ":decimal_class must be BigDecimal, Float, or nil.");
+	}
+   }
+
     if (Qtrue == rb_funcall(ropts, oj_has_key_id, 1, create_id_sym)) {
 	v = rb_hash_lookup(ropts, create_id_sym);
 	if (Qnil == v) {
@@ -1674,6 +1688,7 @@ Init_oj() {
     oj_array_nl_sym = ID2SYM(rb_intern("array_nl"));		rb_gc_register_address(&oj_array_nl_sym);
     oj_ascii_only_sym = ID2SYM(rb_intern("ascii_only"));	rb_gc_register_address(&oj_ascii_only_sym);
     oj_create_additions_sym = ID2SYM(rb_intern("create_additions"));rb_gc_register_address(&oj_create_additions_sym);
+    oj_decimal_class_sym = ID2SYM(rb_intern("decimal_class"));	rb_gc_register_address(&oj_decimal_class_sym);
     oj_hash_class_sym = ID2SYM(rb_intern("hash_class"));	rb_gc_register_address(&oj_hash_class_sym);
     oj_indent_sym = ID2SYM(rb_intern("indent"));		rb_gc_register_address(&oj_indent_sym);
     oj_max_nesting_sym = ID2SYM(rb_intern("max_nesting"));	rb_gc_register_address(&oj_max_nesting_sym);

ext/oj/oj.h

@@ -310,6 +310,7 @@ extern VALUE	oj_array_class_sym;
 extern VALUE	oj_array_nl_sym;
 extern VALUE	oj_ascii_only_sym;
 extern VALUE	oj_create_additions_sym;
+extern VALUE	oj_decimal_class_sym;
 extern VALUE	oj_hash_class_sym;
 extern VALUE	oj_indent_sym;
 extern VALUE	oj_max_nesting_sym;

lib/oj/version.rb

@@ -1,5 +1,5 @@
 
 module Oj
   # Current version of the module.
-  VERSION = '3.10.16'
+  VERSION = '3.10.17'
 end

pages/Modes.md

@@ -94,7 +94,7 @@ information.
 | :array_nl              | String  |         |         |         |         |         |       x |         |
 | :ascii_only            | Boolean |       x |       x |       2 |       2 |       x |       x |         |
 | :auto_define           | Boolean |         |         |         |         |       x |       x |         |
-| :bigdecimal_as_decimal | Boolean |         |         |         |       3 |       x |       x |         |
+| :bigdecimal_as_decimal | Boolean |         |         |       x |       3 |       x |       x |         |
 | :bigdecimal_load       | Boolean |         |         |         |         |         |       x |         |
 | :circular              | Boolean |       x |       x |       x |       x |       x |       x |         |
 | :class_cache           | Boolean |         |         |         |         |       x |       x |         |

pages/Options.md

@@ -66,6 +66,10 @@ Determines how to load decimals.
 
  - `:auto` the most precise for the number of digits is used.
 
+This can also be set with `:decimal_class` when used as a load or
+parse option to match the JSON gem. In that case either `Float`,
+`BigDecimal`, or `nil` can be provided.
+
 ### :circular [Boolean]
 
 Detect circular references while dumping. In :compat mode raise a

test/test_compat.rb

@@ -277,12 +277,19 @@ def test_bignum_compat
   # BigDecimal
   def test_bigdecimal
     # BigDecimals are dumped as strings and can not be restored to the
-    # original value.
+    # original value without using an undocumented feature of the JSON gem.
     json = Oj.dump(BigDecimal('3.14159265358979323846'))
     # 2.4.0 changes the exponent to lowercase
     assert_equal('"0.314159265358979323846e1"', json.downcase)
   end
 
+  def test_bigdecimal_load
+    big = BigDecimal('3.14159265358979323846')
+    # :decimal_class is the undocumented feature.
+    json = Oj.load('3.14159265358979323846', mode: :compat, decimal_class: BigDecimal)
+    assert_equal(big, json)
+  end
+
   def test_infinity
     assert_raises(Oj::ParseError) { Oj.load('Infinity', :mode => :strict) }
     x = Oj.load('Infinity', :mode => :compat)

test/test_rails.rb

@@ -23,4 +23,13 @@ def test_bigdecimal_dump
     assert_equal('0.123e3', json.downcase)
   end
 
+  def test_invalid_encoding
+    assert_raises(EncodingError) {
+      Oj.dump("\"\xf3j", mode: :rails)
+    }
+    assert_raises(EncodingError) {
+      Oj.dump("\xf3j", mode: :rails)
+    }
+  end
+
 end