diff --git a/CHANGELOG.md b/CHANGELOG.md index b3172f3a..13d1b572 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # CHANGELOG +## 3.10.17 - 2020-12-15 + +- The undocumented JSON gem option of `:decimal_class` is now + supported and the default option of `:bigdecimal_load` is also + honored in JSON.parse() and in compat mode. + +- Invalid encoding detection bug fixed for rails. + ## 3.10.16 - 2020-11-10 - Allow escaping any character in :compat mode to match the json gem behavior. diff --git a/ext/oj/dump.c b/ext/oj/dump.c index 105a51ea..c9492e76 100644 --- a/ext/oj/dump.c +++ b/ext/oj/dump.c @@ -174,15 +174,20 @@ hixss_friendly_size(const uint8_t *str, size_t len) { return size - len * (size_t)'0' + check; } -inline static size_t +inline static long rails_xss_friendly_size(const uint8_t *str, size_t len) { - size_t size = 0; + long size = 0; size_t i = len; + uint8_t hi = 0; for (; 0 < i; str++, i--) { size += rails_xss_friendly_chars[*str]; + hi |= *str & 0x80; } - return size - len * (size_t)'0'; + if (0 == hi) { + return size - len * (size_t)'0'; + } + return -(size - len * (size_t)'0'); } inline static size_t @@ -249,7 +254,6 @@ dump_hex(uint8_t c, Out out) { static void raise_invalid_unicode(const char *str, int len, int pos) { - char buf[len + 1]; char c; char code[32]; char *cp = code; @@ -268,8 +272,7 @@ raise_invalid_unicode(const char *str, int len, int pos) { cp--; *cp++ = ']'; *cp = '\0'; - strncpy(buf, str, len); - rb_raise(oj_json_generator_error_class, "Invalid Unicode %s at %d in '%s'", code, pos, buf); + rb_raise(oj_json_generator_error_class, "Invalid Unicode %s at %d", code, pos); } static const char* @@ -767,6 +770,7 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) { size_t size; char *cmap; const char *orig = str; + bool has_hi = false; switch (out->opts->escape_mode) { case NLEsc: @@ -785,10 +789,19 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) { cmap = hixss_friendly_chars; size = hixss_friendly_size((uint8_t*)str, cnt); break; - case RailsXEsc: + case RailsXEsc: { + long sz; + cmap = rails_xss_friendly_chars; - size = rails_xss_friendly_size((uint8_t*)str, cnt); + sz = rails_xss_friendly_size((uint8_t*)str, cnt); + if (sz < 0) { + has_hi = true; + size = (size_t)-sz; + } else { + size = (size_t)sz; + } break; + } case RailsEsc: cmap = rails_friendly_chars; size = rails_friendly_size((uint8_t*)str, cnt); @@ -812,7 +825,7 @@ oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out out) { str++; is_sym = 0; // just to make sure } - if (cnt == size) { + if (cnt == size && !has_hi) { if (is_sym) { *out->cur++ = ':'; } diff --git a/ext/oj/mimic_json.c b/ext/oj/mimic_json.c index db22cb8e..3119fd4f 100644 --- a/ext/oj/mimic_json.c +++ b/ext/oj/mimic_json.c @@ -510,8 +510,6 @@ mimic_parse_core(int argc, VALUE *argv, VALUE self, bool bang) { pi.options.create_ok = No; pi.options.allow_nan = (bang ? Yes : No); pi.options.nilnil = No; - //pi.options.bigdec_load = FloatDec; - pi.options.bigdec_load = RubyDec; pi.options.mode = CompatMode; pi.max_depth = 100; @@ -561,6 +559,16 @@ mimic_parse_core(int argc, VALUE *argv, VALUE self, bool bang) { pi.options.array_class = v; } } + if (Qtrue == rb_funcall(ropts, oj_has_key_id, 1, oj_decimal_class_sym)) { + v = rb_hash_lookup(ropts, oj_decimal_class_sym); + if (rb_cFloat == v) { + pi.options.bigdec_load = FloatDec; + } else if (oj_bigdecimal_class == v) { + pi.options.bigdec_load = BigDec; + } else if (Qnil == v) { + pi.options.bigdec_load = AutoDec; + } + } v = rb_hash_lookup(ropts, oj_max_nesting_sym); if (Qtrue == v) { pi.max_depth = 100; diff --git a/ext/oj/oj.c b/ext/oj/oj.c index a322c7d9..e55b41c8 100644 --- a/ext/oj/oj.c +++ b/ext/oj/oj.c @@ -88,6 +88,7 @@ VALUE oj_slash_string; VALUE oj_allow_nan_sym; VALUE oj_array_class_sym; VALUE oj_create_additions_sym; +VALUE oj_decimal_class_sym; VALUE oj_hash_class_sym; VALUE oj_indent_sym; VALUE oj_object_class_sym; @@ -581,6 +582,19 @@ oj_parse_options(VALUE ropts, Options copts) { rb_raise(rb_eArgError, ":bigdecimal_load must be :bigdecimal, :float, or :auto."); } } + if (Qtrue == rb_funcall(ropts, oj_has_key_id, 1, oj_decimal_class_sym)) { + v = rb_hash_lookup(ropts, oj_decimal_class_sym); + if (rb_cFloat == v) { + copts->bigdec_load = FloatDec; + } else if (oj_bigdecimal_class == v) { + copts->bigdec_load = BigDec; + } else if (Qnil == v) { + copts->bigdec_load = AutoDec; + } else { + rb_raise(rb_eArgError, ":decimal_class must be BigDecimal, Float, or nil."); + } + } + if (Qtrue == rb_funcall(ropts, oj_has_key_id, 1, create_id_sym)) { v = rb_hash_lookup(ropts, create_id_sym); if (Qnil == v) { @@ -1674,6 +1688,7 @@ Init_oj() { oj_array_nl_sym = ID2SYM(rb_intern("array_nl")); rb_gc_register_address(&oj_array_nl_sym); oj_ascii_only_sym = ID2SYM(rb_intern("ascii_only")); rb_gc_register_address(&oj_ascii_only_sym); oj_create_additions_sym = ID2SYM(rb_intern("create_additions"));rb_gc_register_address(&oj_create_additions_sym); + oj_decimal_class_sym = ID2SYM(rb_intern("decimal_class")); rb_gc_register_address(&oj_decimal_class_sym); oj_hash_class_sym = ID2SYM(rb_intern("hash_class")); rb_gc_register_address(&oj_hash_class_sym); oj_indent_sym = ID2SYM(rb_intern("indent")); rb_gc_register_address(&oj_indent_sym); oj_max_nesting_sym = ID2SYM(rb_intern("max_nesting")); rb_gc_register_address(&oj_max_nesting_sym); diff --git a/ext/oj/oj.h b/ext/oj/oj.h index b059420d..002f6f60 100644 --- a/ext/oj/oj.h +++ b/ext/oj/oj.h @@ -310,6 +310,7 @@ extern VALUE oj_array_class_sym; extern VALUE oj_array_nl_sym; extern VALUE oj_ascii_only_sym; extern VALUE oj_create_additions_sym; +extern VALUE oj_decimal_class_sym; extern VALUE oj_hash_class_sym; extern VALUE oj_indent_sym; extern VALUE oj_max_nesting_sym; diff --git a/lib/oj/version.rb b/lib/oj/version.rb index e40d99b0..54cfa408 100644 --- a/lib/oj/version.rb +++ b/lib/oj/version.rb @@ -1,5 +1,5 @@ module Oj # Current version of the module. - VERSION = '3.10.16' + VERSION = '3.10.17' end diff --git a/pages/Modes.md b/pages/Modes.md index 190a90fd..9f86eab3 100644 --- a/pages/Modes.md +++ b/pages/Modes.md @@ -94,7 +94,7 @@ information. | :array_nl | String | | | | | | x | | | :ascii_only | Boolean | x | x | 2 | 2 | x | x | | | :auto_define | Boolean | | | | | x | x | | -| :bigdecimal_as_decimal | Boolean | | | | 3 | x | x | | +| :bigdecimal_as_decimal | Boolean | | | x | 3 | x | x | | | :bigdecimal_load | Boolean | | | | | | x | | | :circular | Boolean | x | x | x | x | x | x | | | :class_cache | Boolean | | | | | x | x | | diff --git a/pages/Options.md b/pages/Options.md index 48c8d518..07f8ed6a 100644 --- a/pages/Options.md +++ b/pages/Options.md @@ -66,6 +66,10 @@ Determines how to load decimals. - `:auto` the most precise for the number of digits is used. +This can also be set with `:decimal_class` when used as a load or +parse option to match the JSON gem. In that case either `Float`, +`BigDecimal`, or `nil` can be provided. + ### :circular [Boolean] Detect circular references while dumping. In :compat mode raise a diff --git a/test/test_compat.rb b/test/test_compat.rb index b292d66c..68b094cc 100755 --- a/test/test_compat.rb +++ b/test/test_compat.rb @@ -277,12 +277,19 @@ def test_bignum_compat # BigDecimal def test_bigdecimal # BigDecimals are dumped as strings and can not be restored to the - # original value. + # original value without using an undocumented feature of the JSON gem. json = Oj.dump(BigDecimal('3.14159265358979323846')) # 2.4.0 changes the exponent to lowercase assert_equal('"0.314159265358979323846e1"', json.downcase) end + def test_bigdecimal_load + big = BigDecimal('3.14159265358979323846') + # :decimal_class is the undocumented feature. + json = Oj.load('3.14159265358979323846', mode: :compat, decimal_class: BigDecimal) + assert_equal(big, json) + end + def test_infinity assert_raises(Oj::ParseError) { Oj.load('Infinity', :mode => :strict) } x = Oj.load('Infinity', :mode => :compat) diff --git a/test/test_rails.rb b/test/test_rails.rb index aa529013..20e2da0d 100755 --- a/test/test_rails.rb +++ b/test/test_rails.rb @@ -23,4 +23,13 @@ def test_bigdecimal_dump assert_equal('0.123e3', json.downcase) end + def test_invalid_encoding + assert_raises(EncodingError) { + Oj.dump("\"\xf3j", mode: :rails) + } + assert_raises(EncodingError) { + Oj.dump("\xf3j", mode: :rails) + } + end + end