Permalink
Browse files

ready for 0.6.0 release

  • Loading branch information...
1 parent 8711c85 commit 666920663f3cc0d9b766ba47267172ffb0c7e73e Peter Ohler committed Feb 22, 2012
Showing with 197 additions and 44 deletions.
  1. +90 −1 README.md
  2. +27 −3 ext/oj/dump.c
  3. +17 −0 ext/oj/load.c
  4. +0 −20 notes
  5. +45 −20 test/perf_simple.rb
  6. +18 −0 test/simple.rb
View
@@ -26,7 +26,7 @@ A fast JSON parser and Object marshaller as a Ruby gem.
- to_json() method called if the Object responds to to_json
-- any Object with variables can be dumped correctly
+- almost any Object can be dumped, including Exceptions (not including Thread, Mutex and Objects that only make sense within a process)
- default options have been added
@@ -59,6 +59,95 @@ Coming soon: A JSON stream parser.
Oj is compatible with Ruby 1.8.7, 1.9.2, 1.9.3, JRuby, and RBX.
+## <a name="compare">Comparisons</a>
+
+The following table shows the difference is speeds between several
+serialization packages. The tests had to be scaled back due to limitation of
+some of the gems. I finally gave up trying to get JSON to serialize without
+errors with Ruby 1.9.3. It had internal errors on anything other than a simple
+JSON structure. The errors encountered were:
+
+- MessagePack fails to convert Bignum to JSON
+
+- JSON Pure and Ext fails to serialize any numbers or Objects with the to_json() method
+
+Options were added to the test/perf_simple.rb test to run the test without
+Object encoding and without Bignums. There is also an option for shallow JSON
+so that JSON Pure and Ext can be compared.
+
+None of the packages except Oj were able to serialize Ruby Objects that did
+not have a to_json() method or were of the 7 native JSON types.
+
+It is also worth noting that although Oj is slightly behind MessagePack for
+parsing, Oj serialization is much faster than MessagePack even though Oj uses
+human readable JSON vs the binary MessagePack format.
+
+The results:
+
+with Object and Bignum encoding:
+
+ 100000 Oj.load()s in 1.456 seconds or 68.7 loads/msec
+ 100000 Yajl::Parser.parse()s in 2.681 seconds or 37.3 parses/msec
+ 100000 JSON::Ext::Parser parse()s in 2.804 seconds or 35.7 parses/msec
+ 100000 JSON::Pure::Parser parse()s in 27.494 seconds or 3.6 parses/msec
+ MessagePack failed: RangeError: bignum too big to convert into `unsigned long long'
+ 100000 Ox.load()s in 3.165 seconds or 31.6 loads/msec
+ Parser results:
+ gem seconds parses/msec X faster than JSON::Pure (higher is better)
+ oj 1.456 68.7 18.9
+ yajl 2.681 37.3 10.3
+ msgpack failed to generate JSON
+ pure 27.494 3.6 1.0
+ ext 2.804 35.7 9.8
+ ox 3.165 31.6 8.7
+
+ 100000 Oj.dump()s in 0.484 seconds or 206.7 dumps/msec
+ 100000 Yajl::Encoder.encode()s in 2.167 seconds or 46.2 encodes/msec
+ JSON::Ext failed: TypeError: wrong argument type JSON::Pure::Generator::State (expected Data)
+ JSON::Pure failed: TypeError: wrong argument type JSON::Pure::Generator::State (expected Data)
+ MessagePack failed: RangeError: bignum too big to convert into `unsigned long long'
+ 100000 Ox.dump()s in 0.554 seconds or 180.4 dumps/msec
+ Parser results:
+ gem seconds dumps/msec X faster than Yajl (higher is better)
+ oj 0.484 206.7 4.5
+ yajl 2.167 46.2 1.0
+ msgpack failed to generate JSON
+ pure failed to generate JSON
+ ext failed to generate JSON
+ ox 0.554 180.4 3.9
+
+without Objects or numbers (for JSON Pure) JSON:
+
+ 100000 Oj.load()s in 0.739 seconds or 135.3 loads/msec
+ 100000 Yajl::Parser.parse()s in 1.421 seconds or 70.4 parses/msec
+ 100000 JSON::Ext::Parser parse()s in 1.512 seconds or 66.2 parses/msec
+ 100000 JSON::Pure::Parser parse()s in 16.953 seconds or 5.9 parses/msec
+ 100000 MessagePack.unpack()s in 0.635 seconds or 157.6 packs/msec
+ 100000 Ox.load()s in 0.971 seconds or 103.0 loads/msec
+ Parser results:
+ gem seconds parses/msec X faster than JSON::Pure (higher is better)
+ oj 0.739 135.3 22.9
+ yajl 1.421 70.4 11.9
+ msgpack 0.635 157.6 26.7
+ pure 16.953 5.9 1.0
+ ext 1.512 66.2 11.2
+ ox 0.971 103.0 17.5
+
+ 100000 Oj.dump()s in 0.174 seconds or 575.1 dumps/msec
+ 100000 Yajl::Encoder.encode()s in 0.729 seconds or 137.2 encodes/msec
+ 100000 JSON::Ext generate()s in 7.171 seconds or 13.9 generates/msec
+ 100000 JSON::Pure generate()s in 7.219 seconds or 13.9 generates/msec
+ 100000 Msgpack()s in 0.299 seconds or 334.8 unpacks/msec
+ 100000 Ox.dump()s in 0.210 seconds or 475.8 dumps/msec
+ Parser results:
+ gem seconds dumps/msec X faster than JSON::Pure (higher is better)
+ oj 0.174 575.1 41.5
+ yajl 0.729 137.2 9.9
+ msgpack 0.299 334.8 24.2
+ pure 7.219 13.9 1.0
+ ext 1.512 66.2 4.8
+ ox 0.210 475.8 34.3
+
### Simple JSON Writing and Parsing:
require 'oj'
View
@@ -79,6 +79,7 @@ static void dump_nil(Out out);
static void dump_true(Out out);
static void dump_false(Out out);
static void dump_fixnum(VALUE obj, Out out);
+static void dump_bignum(VALUE obj, Out out);
static void dump_float(VALUE obj, Out out);
static void dump_cstr(const char *str, int cnt, Out out);
static void dump_hex(u_char c, Out out);
@@ -255,6 +256,19 @@ dump_fixnum(VALUE obj, Out out) {
}
static void
+dump_bignum(VALUE obj, Out out) {
+ VALUE rs = rb_big2str(obj, 10);
+ int cnt = (int)RSTRING_LEN(rs);
+
+ if (out->end - out->cur <= (long)cnt) {
+ grow(out, cnt);
+ }
+ memcpy(out->cur, StringValuePtr(rs), cnt);
+ out->cur += cnt;
+ *out->cur = '\0';
+}
+
+static void
dump_float(VALUE obj, Out out) {
char buf[64];
char *b;
@@ -550,8 +564,13 @@ dump_attr_cb(ID key, VALUE value, Out out) {
if (out->end - out->cur <= (long)size) {
grow(out, size);
}
+ if ('@' == *attr) {
+ attr++;
+ } else {
+ // TBD handle unusual exception mesg data
+ }
fill_indent(out, depth);
- dump_cstr(attr + 1, (int)strlen(attr) - 1, out);
+ dump_cstr(attr, (int)strlen(attr) - 1, out);
*out->cur++ = ':';
dump_val(value, depth, out);
out->depth = depth;
@@ -614,7 +633,12 @@ dump_obj_attrs(VALUE obj, int with_class, int depth, Out out) {
vid = rb_to_id(*np);
fill_indent(out, d2);
attr = rb_id2name(vid);
- dump_cstr(attr + 1, (int)strlen(attr) - 1, out);
+ if ('@' == *attr) {
+ attr++;
+ } else {
+ // TBD handle unusual exception mesg data
+ }
+ dump_cstr(attr, (int)strlen(attr) - 1, out);
*out->cur++ = ':';
dump_val(rb_ivar_get(obj, vid), d2, out);
if (out->end - out->cur <= 2) {
@@ -639,7 +663,7 @@ dump_val(VALUE obj, int depth, Out out) {
case T_FALSE: dump_false(out); break;
case T_FIXNUM: dump_fixnum(obj, out); break;
case T_FLOAT: dump_float(obj, out); break;
- case T_BIGNUM: break; // TBD
+ case T_BIGNUM: dump_bignum(obj, out); break;
case T_STRING: dump_str(obj, out); break;
case T_SYMBOL: dump_sym(obj, out); break;
case T_ARRAY: dump_array(obj, depth, out); break;
View
@@ -261,14 +261,18 @@ read_str(ParseInfo pi) {
return s;
}
+#define NUM_MAX (FIXNUM_MAX >> 8)
+
static VALUE
read_num(ParseInfo pi) {
+ char *start = pi->s;
int64_t n = 0;
long a = 0;
long div = 1;
long e = 0;
int neg = 0;
int eneg = 0;
+ int big = 0;
if ('-' == *pi->s) {
pi->s++;
@@ -278,6 +282,19 @@ read_num(ParseInfo pi) {
}
for (; '0' <= *pi->s && *pi->s <= '9'; pi->s++) {
n = n * 10 + (*pi->s - '0');
+ if (NUM_MAX <= n) {
+ big = 1;
+ }
+ }
+ if (big) {
+ char c = *pi->s;
+ VALUE num;
+
+ *pi->s = '\0';
+ num = rb_cstr_to_inum(start, 10, 0);
+ *pi->s = c;
+
+ return num;
}
if ('.' == *pi->s) {
pi->s++;
View
20 notes
@@ -5,37 +5,17 @@
- next
- - make sure dump buffers are not exceeded
- - test dump Exception
- implement load mode
- stream
- load
- dump
- load
- - todo
- - bignum
- dump
- - bignum
- multibyte group encoding
- object and other types of object dumping
-- dump
- - options
- - object or simple (needed for Hash)
- - call sjon on objects, skip, or raise
- - strict - raise
- - lazy / nil
- - tolerant / best effort
- - use to_json if respond_to
- - walk attributes
-
-
-- write a stream parser and writer
-
-- serialize/deserialize any object
- - add class as first key in an object hash or make class and vars keys only (id for circular later)
- support circular object encoding
View
@@ -47,29 +47,44 @@ def to_msgpack(out)
$indent = 2
$iter = 10000
$with_object = true
+$with_bignum = true
+$with_nums = true
opts = OptionParser.new
opts.on("-c", "--count [Int]", Integer, "iterations") { |i| $iter = i }
opts.on("-i", "--indent [Int]", Integer, "indentation") { |i| $indent = i }
opts.on("-o", "without objects") { $with_object = false }
+opts.on("-b", "without bignum") { $with_bignum = false }
+opts.on("-n", "without numbers") { $with_nums = false }
opts.on("-h", "--help", "Show this display") { puts opts; Process.exit!(0) }
files = opts.parse(ARGV)
-obj = {
- 'a' => 'Alpha',
- 'b' => true,
- 'c' => 12345,
- 'd' => [ true, [false, [12345, nil], 3.967, ['something', false], nil]],
- 'e' => { 'one' => 1, 'two' => 2 },
- 'f' => nil
+if $with_nums
+ obj = {
+ 'a' => 'Alpha',
+ 'b' => true,
+ 'c' => 12345,
+ 'd' => [ true, [false, [12345, nil], 3.967, ['something', false], nil]],
+ 'e' => { 'one' => 1, 'two' => 2 },
+ 'f' => nil,
}
-obj['g'] = Jazz.new() if $with_object
+ obj['g'] = Jazz.new() if $with_object
+ obj['h'] = 12345678901234567890123456789 if $with_bignum
+else
+ obj = {
+ 'a' => 'Alpha',
+ 'b' => true,
+ 'c' => '12345',
+ 'd' => [ true, [false, ['12345', nil], '3.967', ['something', false], nil]],
+ 'e' => { 'one' => '1', 'two' => '2' },
+ 'f' => nil,
+ }
+end
Oj.default_options = { :indent => $indent, :mode => :object }
s = Oj.dump(obj)
-mp = MessagePack.pack(obj)
xml = Ox.dump(obj, :indent => $indent)
puts
@@ -136,6 +151,7 @@ def to_msgpack(out)
end
begin
+ mp = MessagePack.pack(obj)
start = Time.now
$iter.times do
MessagePack.unpack(mp)
@@ -148,7 +164,7 @@ def to_msgpack(out)
parse_results[:msgpack] = dt
puts "%d MessagePack.unpack()s in %0.3f seconds or %0.1f packs/msec" % [$iter, dt, $iter/dt/1000.0]
rescue Exception => e
- puts "JSON::Pure failed: #{e.class}: #{e.message}"
+ puts "MessagePack failed: #{e.class}: #{e.message}"
end
start = Time.now
@@ -162,6 +178,10 @@ def to_msgpack(out)
puts "Parser results:"
puts "gem seconds parses/msec X faster than #{base_name} (higher is better)"
parse_results.each do |name,dt|
+ if 0.0 == dt
+ puts "#{name} failed to generate JSON"
+ next
+ end
puts "%-7s %6.3f %5.1f %4.1f" % [name, dt, $iter/dt/1000.0, base_dt/dt]
end
@@ -229,17 +249,22 @@ def to_msgpack(out)
puts "JSON::Pure failed: #{e.class}: #{e.message}"
end
-start = Time.now
-$iter.times do
- MessagePack.pack(obj)
-end
-dt = Time.now - start
-if base_dt < dt
- base_dt = dt
- base_name = 'MessagePack'
+begin
+ start = Time.now
+ $iter.times do
+ MessagePack.pack(obj)
+ end
+ dt = Time.now - start
+ if base_dt < dt
+ base_dt = dt
+ base_name = 'MessagePack'
+ end
+ parse_results[:msgpack] = dt
+ puts "%d Msgpack()s in %0.3f seconds or %0.1f unpacks/msec" % [$iter, dt, $iter/dt/1000.0]
+rescue Exception => e
+ parse_results[:msgpack] = 0.0
+ puts "MessagePack failed: #{e.class}: #{e.message}"
end
-parse_results[:msgpack] = dt
-puts "%d Msgpack()s in %0.3f seconds or %0.1f unpacks/msec" % [$iter, dt, $iter/dt/1000.0]
start = Time.now
$iter.times do
View
@@ -79,6 +79,10 @@ def test_fixnum
dump_and_load(1, false)
end
+ def test_bignum
+ dump_and_load(12345678901234567890123456789, true)
+ end
+
def test_float
dump_and_load(0.0, false)
dump_and_load(12345.6789, false)
@@ -179,6 +183,20 @@ def test_object
assert_equal('null', json)
end
+ def test_exception
+ err = nil
+ begin
+ raise StandardError.new('A Message')
+ rescue Exception => e
+ err = e
+ end
+ json = Oj.dump(err, :mode => :object, :indent => 2)
+ e2 = Oj.load(json, :mode => :strict)
+ assert_equal(err.class.to_s, e2['*'])
+ assert_equal(err.message, e2['mesg'])
+ assert_equal(err.backtrace, e2['bt'])
+ end
+
def dump_and_load(obj, trace=false)
json = Oj.dump(obj, :indent => 2)
puts json if trace

0 comments on commit 6669206

Please sign in to comment.