Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

faster dumping

  • Loading branch information...
commit dc55a9615b59bac93b8a811d6108629af005e2e2 1 parent 6ed67bb
Peter Ohler authored
15 README.md
Source Rendered
@@ -32,20 +32,15 @@ A fast JSON parser and Object marshaller as a Ruby gem.
32 32
33 33 ## <a name="release">Release Notes</a>
34 34
35   -### Release 2.0.9
36   -
37   - - Fixed problem with INFINITY with CentOS and Ruby 2.0.0. There are some header file conflicts so a different INFINITY was used.
  35 +### Release 2.0.10
38 36
39   -### Release 2.0.8
  37 + - Tweaked dump calls by reducing preallocation. Speeds are now several times faster for smaller objects.
40 38
41   - - Added :bigdecimal_load option that forces all decimals in a JSON string to be read as BigDecimals instead of as
42   - Floats. This is useful if precision is important.
  39 + - TBD Fixed Windows compile error with Ruby 2.0.0.
43 40
44   - - Worked around bug in active_support 2.3.x where BigDecimal.as_json() returned self instead of a JSON primitive. Of
45   - course that creates a loop and blows the stack. Oj ignores the as_json() for any object that returns itself and
46   - instead encodes the object as it sees fit which is usually what is expected.
  41 +### Release 2.0.9
47 42
48   - - All tests pass with Ruby 2.0.0-p0. Had to modify Exception encoding slightly.
  43 + - Fixed problem with INFINITY with CentOS and Ruby 2.0.0. There are some header file conflicts so a different INFINITY was used.
49 44
50 45 ## <a name="description">Description</a>
51 46
84 ext/oj/dump.c
@@ -50,19 +50,6 @@
50 50
51 51 typedef unsigned long ulong;
52 52
53   -typedef struct _Out {
54   - char *buf;
55   - char *end;
56   - char *cur;
57   - Cache8 circ_cache;
58   - slot_t circ_cnt;
59   - int indent;
60   - int depth; // used by dump_hash
61   - Options opts;
62   - uint32_t hash_cnt;
63   -} *Out;
64   -
65   -static void dump_obj_to_json(VALUE obj, Options copts, Out out);
66 53 static void raise_strict(VALUE obj);
67 54 static void dump_val(VALUE obj, int depth, Out out);
68 55 static void dump_nil(Out out);
@@ -108,7 +95,6 @@ static void grow(Out out, size_t len);
108 95 static size_t hibit_friendly_size(const uint8_t *str, size_t len);
109 96 static size_t ascii_friendly_size(const uint8_t *str, size_t len);
110 97
111   -static void dump_leaf_to_json(Leaf leaf, Options copts, Out out);
112 98 static void dump_leaf(Leaf leaf, int depth, Out out);
113 99 static void dump_leaf_str(Leaf leaf, Out out);
114 100 static void dump_leaf_fixnum(Leaf leaf, Out out);
@@ -215,8 +201,14 @@ grow(Out out, size_t len) {
215 201 if (size <= len * 2 + pos) {
216 202 size += len;
217 203 }
218   - buf = REALLOC_N(out->buf, char, (size + 10));
219   - if (0 == buf) { // 1 extra for terminator character plus extra (paranoid)
  204 + if (out->allocated) {
  205 + buf = REALLOC_N(out->buf, char, (size + 10));
  206 + } else {
  207 + buf = ALLOC_N(char, (size + 10));
  208 + out->allocated = 1;
  209 + memcpy(buf, out->buf, out->end - out->buf);
  210 + }
  211 + if (0 == buf) {
220 212 rb_raise(rb_eNoMemError, "Failed to create string. [%d:%s]\n", ENOSPC, strerror(ENOSPC));
221 213 }
222 214 out->buf = buf;
@@ -1650,10 +1642,13 @@ dump_val(VALUE obj, int depth, Out out) {
1650 1642 }
1651 1643 }
1652 1644
1653   -static void
1654   -dump_obj_to_json(VALUE obj, Options copts, Out out) {
1655   - out->buf = ALLOC_N(char, 4096);
1656   - out->end = out->buf + 4085; // 1 less than end plus extra for possible errors
  1645 +void
  1646 +oj_dump_obj_to_json(VALUE obj, Options copts, Out out) {
  1647 + if (0 == out->buf) {
  1648 + out->buf = ALLOC_N(char, 4096);
  1649 + out->end = out->buf + 4085; // 1 less than end plus extra for possible errors
  1650 + out->allocated = 1;
  1651 + }
1657 1652 out->cur = out->buf;
1658 1653 out->circ_cnt = 0;
1659 1654 out->opts = copts;
@@ -1668,22 +1663,17 @@ dump_obj_to_json(VALUE obj, Options copts, Out out) {
1668 1663 }
1669 1664 }
1670 1665
1671   -char*
1672   -oj_write_obj_to_str(VALUE obj, Options copts) {
1673   - struct _Out out;
1674   -
1675   - dump_obj_to_json(obj, copts, &out);
1676   -
1677   - return out.buf;
1678   -}
1679   -
1680 1666 void
1681 1667 oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
  1668 + char buf[4096];
1682 1669 struct _Out out;
1683 1670 size_t size;
1684 1671 FILE *f;
1685 1672
1686   - dump_obj_to_json(obj, copts, &out);
  1673 + out.buf = buf;
  1674 + out.end = buf + sizeof(buf) - 10;
  1675 + out.allocated = 0;
  1676 + oj_dump_obj_to_json(obj, copts, &out);
1687 1677 size = out.cur - out.buf;
1688 1678 if (0 == (f = fopen(path, "w"))) {
1689 1679 rb_raise(rb_eIOError, "%s\n", strerror(errno));
@@ -1693,7 +1683,9 @@ oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
1693 1683
1694 1684 rb_raise(rb_eIOError, "Write failed. [%d:%s]\n", err, strerror(err));
1695 1685 }
1696   - xfree(out.buf);
  1686 + if (out.allocated) {
  1687 + xfree(out.buf);
  1688 + }
1697 1689 fclose(f);
1698 1690 }
1699 1691
@@ -1872,10 +1864,13 @@ dump_leaf(Leaf leaf, int depth, Out out) {
1872 1864 }
1873 1865 }
1874 1866
1875   -static void
1876   -dump_leaf_to_json(Leaf leaf, Options copts, Out out) {
1877   - out->buf = ALLOC_N(char, 4096);
1878   - out->end = out->buf + 4085; // 10 less than end plus extra for possible errors
  1867 +void
  1868 +oj_dump_leaf_to_json(Leaf leaf, Options copts, Out out) {
  1869 + if (0 == out->buf) {
  1870 + out->buf = ALLOC_N(char, 4096);
  1871 + out->end = out->buf + 4085; // 1 less than end plus extra for possible errors
  1872 + out->allocated = 1;
  1873 + }
1879 1874 out->cur = out->buf;
1880 1875 out->circ_cnt = 0;
1881 1876 out->opts = copts;
@@ -1884,22 +1879,17 @@ dump_leaf_to_json(Leaf leaf, Options copts, Out out) {
1884 1879 dump_leaf(leaf, 0, out);
1885 1880 }
1886 1881
1887   -char*
1888   -oj_write_leaf_to_str(Leaf leaf, Options copts) {
1889   - struct _Out out;
1890   -
1891   - dump_leaf_to_json(leaf, copts, &out);
1892   -
1893   - return out.buf;
1894   -}
1895   -
1896 1882 void
1897 1883 oj_write_leaf_to_file(Leaf leaf, const char *path, Options copts) {
  1884 + char buf[4096];
1898 1885 struct _Out out;
1899 1886 size_t size;
1900 1887 FILE *f;
1901 1888
1902   - dump_leaf_to_json(leaf, copts, &out);
  1889 + out.buf = buf;
  1890 + out.end = buf + sizeof(buf) - 10;
  1891 + out.allocated = 0;
  1892 + oj_dump_leaf_to_json(leaf, copts, &out);
1903 1893 size = out.cur - out.buf;
1904 1894 if (0 == (f = fopen(path, "w"))) {
1905 1895 rb_raise(rb_eIOError, "%s\n", strerror(errno));
@@ -1909,6 +1899,8 @@ oj_write_leaf_to_file(Leaf leaf, const char *path, Options copts) {
1909 1899
1910 1900 rb_raise(rb_eIOError, "Write failed. [%d:%s]\n", err, strerror(err));
1911 1901 }
1912   - xfree(out.buf);
  1902 + if (out.allocated) {
  1903 + xfree(out.buf);
  1904 + }
1913 1905 fclose(f);
1914 1906 }
15 ext/oj/fast.c
@@ -1571,13 +1571,20 @@ doc_dump(int argc, VALUE *argv, VALUE self) {
1571 1571 }
1572 1572 }
1573 1573 if (0 != (leaf = get_doc_leaf(doc, path))) {
1574   - char *json;
1575 1574 VALUE rjson;
1576 1575
1577 1576 if (0 == filename) {
1578   - json = oj_write_leaf_to_str(leaf, &oj_default_options);
1579   - rjson = rb_str_new2(json);
1580   - xfree(json);
  1577 + char buf[4096];
  1578 + struct _Out out;
  1579 +
  1580 + out.buf = buf;
  1581 + out.end = buf + sizeof(buf) - 10;
  1582 + out.allocated = 0;
  1583 + oj_dump_leaf_to_json(leaf, &oj_default_options, &out);
  1584 + rjson = rb_str_new2(out.buf);
  1585 + if (out.allocated) {
  1586 + xfree(out.buf);
  1587 + }
1581 1588 } else {
1582 1589 oj_write_leaf_to_file(leaf, filename, &oj_default_options);
1583 1590 rjson = Qnil;
50 ext/oj/oj.c
@@ -573,22 +573,28 @@ strict_load(VALUE self, VALUE doc) {
573 573 */
574 574 static VALUE
575 575 dump(int argc, VALUE *argv, VALUE self) {
576   - char *json;
  576 + char buf[4096];
  577 + struct _Out out;
577 578 struct _Options copts = oj_default_options;
578 579 VALUE rstr;
579 580
580 581 if (2 == argc) {
581 582 parse_options(argv[1], &copts);
582 583 }
583   - if (0 == (json = oj_write_obj_to_str(*argv, &copts))) {
  584 + out.buf = buf;
  585 + out.end = buf + sizeof(buf) - 10;
  586 + out.allocated = 0;
  587 + oj_dump_obj_to_json(*argv, &copts, &out);
  588 + if (0 == out.buf) {
584 589 rb_raise(rb_eNoMemError, "Not enough memory.");
585 590 }
586   - rstr = rb_str_new2(json);
  591 + rstr = rb_str_new2(out.buf);
587 592 #if HAS_ENCODING_SUPPORT
588 593 rb_enc_associate(rstr, oj_utf8_encoding);
589 594 #endif
590   - xfree(json);
591   -
  595 + if (out.allocated) {
  596 + xfree(out.buf);
  597 + }
592 598 return rstr;
593 599 }
594 600
@@ -698,14 +704,19 @@ saj_parse(int argc, VALUE *argv, VALUE self) {
698 704
699 705 static VALUE
700 706 mimic_dump(int argc, VALUE *argv, VALUE self) {
701   - char *json;
  707 + char buf[4096];
  708 + struct _Out out;
702 709 struct _Options copts = oj_default_options;
703 710 VALUE rstr;
704 711
705   - if (0 == (json = oj_write_obj_to_str(*argv, &copts))) {
  712 + out.buf = buf;
  713 + out.end = buf + sizeof(buf) - 10;
  714 + out.allocated = 0;
  715 + oj_dump_obj_to_json(*argv, &copts, &out);
  716 + if (0 == out.buf) {
706 717 rb_raise(rb_eNoMemError, "Not enough memory.");
707 718 }
708   - rstr = rb_str_new2(json);
  719 + rstr = rb_str_new2(out.buf);
709 720 #if HAS_ENCODING_SUPPORT
710 721 rb_enc_associate(rstr, oj_utf8_encoding);
711 722 #endif
@@ -717,8 +728,9 @@ mimic_dump(int argc, VALUE *argv, VALUE self) {
717 728 rb_funcall2(io, oj_write_id, 1, args);
718 729 rstr = io;
719 730 }
720   - xfree(json);
721   -
  731 + if (out.allocated) {
  732 + xfree(out.buf);
  733 + }
722 734 return rstr;
723 735 }
724 736
@@ -786,9 +798,13 @@ mimic_dump_load(int argc, VALUE *argv, VALUE self) {
786 798
787 799 static VALUE
788 800 mimic_generate_core(int argc, VALUE *argv, Options copts) {
789   - char *json;
790   - VALUE rstr;
  801 + char buf[4096];
  802 + struct _Out out;
  803 + VALUE rstr;
791 804
  805 + out.buf = buf;
  806 + out.end = buf + sizeof(buf) - 10;
  807 + out.allocated = 0;
792 808 if (2 == argc && Qnil != argv[1]) {
793 809 struct _DumpOpts dump_opts;
794 810 VALUE ropts = argv[1];
@@ -841,15 +857,17 @@ mimic_generate_core(int argc, VALUE *argv, Options copts) {
841 857 // :allow_nan is not supported as Oj always allows_nan
842 858 // :max_nesting is always set to 100
843 859 }
844   - if (0 == (json = oj_write_obj_to_str(*argv, copts))) {
  860 + oj_dump_obj_to_json(*argv, copts, &out);
  861 + if (0 == out.buf) {
845 862 rb_raise(rb_eNoMemError, "Not enough memory.");
846 863 }
847   - rstr = rb_str_new2(json);
  864 + rstr = rb_str_new2(out.buf);
848 865 #if HAS_ENCODING_SUPPORT
849 866 rb_enc_associate(rstr, oj_utf8_encoding);
850 867 #endif
851   - xfree(json);
852   -
  868 + if (out.allocated) {
  869 + xfree(out.buf);
  870 + }
853 871 return rstr;
854 872 }
855 873
18 ext/oj/oj.h
@@ -50,6 +50,7 @@ extern "C" {
50 50 #include <pthread.h>
51 51 #endif
52 52 #include "cache.h"
  53 +#include "cache8.h"
53 54
54 55 #ifdef RUBINIUS_RUBY
55 56 #undef T_RATIONAL
@@ -116,6 +117,19 @@ typedef struct _Options {
116 117 DumpOpts dump_opts;
117 118 } *Options;
118 119
  120 +typedef struct _Out {
  121 + char *buf;
  122 + char *end;
  123 + char *cur;
  124 + Cache8 circ_cache;
  125 + slot_t circ_cnt;
  126 + int indent;
  127 + int depth; // used by dump_hash
  128 + Options opts;
  129 + uint32_t hash_cnt;
  130 + int allocated;
  131 +} *Out;
  132 +
119 133 typedef struct _Odd {
120 134 VALUE clas; // Ruby class
121 135 VALUE create_obj;
@@ -149,9 +163,9 @@ typedef struct _Leaf {
149 163 extern VALUE oj_parse(char *json, Options options);
150 164 extern void oj_saj_parse(VALUE handler, char *json);
151 165
152   -extern char* oj_write_obj_to_str(VALUE obj, Options copts);
  166 +extern void oj_dump_obj_to_json(VALUE obj, Options copts, Out out);
153 167 extern void oj_write_obj_to_file(VALUE obj, const char *path, Options copts);
154   -extern char* oj_write_leaf_to_str(Leaf leaf, Options copts);
  168 +extern void oj_dump_leaf_to_json(Leaf leaf, Options copts, Out out);
155 169 extern void oj_write_leaf_to_file(Leaf leaf, const char *path, Options copts);
156 170
157 171 extern void _oj_raise_error(const char *msg, const char *xml, const char *current, const char* file, int line);
2  lib/oj/version.rb
... ... @@ -1,5 +1,5 @@
1 1
2 2 module Oj
3 3 # Current version of the module.
4   - VERSION = '2.0.10.a1'
  4 + VERSION = '2.0.10.a2'
5 5 end

0 comments on commit dc55a96

Please sign in to comment.
Something went wrong with that request. Please try again.