Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

faster dumping

  • Loading branch information...
commit dc55a9615b59bac93b8a811d6108629af005e2e2 1 parent 6ed67bb
@ohler55 authored
View
15 README.md
@@ -32,20 +32,15 @@ A fast JSON parser and Object marshaller as a Ruby gem.
## <a name="release">Release Notes</a>
-### Release 2.0.9
-
- - Fixed problem with INFINITY with CentOS and Ruby 2.0.0. There are some header file conflicts so a different INFINITY was used.
+### Release 2.0.10
-### Release 2.0.8
+ - Tweaked dump calls by reducing preallocation. Speeds are now several times faster for smaller objects.
- - Added :bigdecimal_load option that forces all decimals in a JSON string to be read as BigDecimals instead of as
- Floats. This is useful if precision is important.
+ - TBD Fixed Windows compile error with Ruby 2.0.0.
- - Worked around bug in active_support 2.3.x where BigDecimal.as_json() returned self instead of a JSON primitive. Of
- course that creates a loop and blows the stack. Oj ignores the as_json() for any object that returns itself and
- instead encodes the object as it sees fit which is usually what is expected.
+### Release 2.0.9
- - All tests pass with Ruby 2.0.0-p0. Had to modify Exception encoding slightly.
+ - Fixed problem with INFINITY with CentOS and Ruby 2.0.0. There are some header file conflicts so a different INFINITY was used.
## <a name="description">Description</a>
View
84 ext/oj/dump.c
@@ -50,19 +50,6 @@
typedef unsigned long ulong;
-typedef struct _Out {
- char *buf;
- char *end;
- char *cur;
- Cache8 circ_cache;
- slot_t circ_cnt;
- int indent;
- int depth; // used by dump_hash
- Options opts;
- uint32_t hash_cnt;
-} *Out;
-
-static void dump_obj_to_json(VALUE obj, Options copts, Out out);
static void raise_strict(VALUE obj);
static void dump_val(VALUE obj, int depth, Out out);
static void dump_nil(Out out);
@@ -108,7 +95,6 @@ static void grow(Out out, size_t len);
static size_t hibit_friendly_size(const uint8_t *str, size_t len);
static size_t ascii_friendly_size(const uint8_t *str, size_t len);
-static void dump_leaf_to_json(Leaf leaf, Options copts, Out out);
static void dump_leaf(Leaf leaf, int depth, Out out);
static void dump_leaf_str(Leaf leaf, Out out);
static void dump_leaf_fixnum(Leaf leaf, Out out);
@@ -215,8 +201,14 @@ grow(Out out, size_t len) {
if (size <= len * 2 + pos) {
size += len;
}
- buf = REALLOC_N(out->buf, char, (size + 10));
- if (0 == buf) { // 1 extra for terminator character plus extra (paranoid)
+ if (out->allocated) {
+ buf = REALLOC_N(out->buf, char, (size + 10));
+ } else {
+ buf = ALLOC_N(char, (size + 10));
+ out->allocated = 1;
+ memcpy(buf, out->buf, out->end - out->buf);
+ }
+ if (0 == buf) {
rb_raise(rb_eNoMemError, "Failed to create string. [%d:%s]\n", ENOSPC, strerror(ENOSPC));
}
out->buf = buf;
@@ -1650,10 +1642,13 @@ dump_val(VALUE obj, int depth, Out out) {
}
}
-static void
-dump_obj_to_json(VALUE obj, Options copts, Out out) {
- out->buf = ALLOC_N(char, 4096);
- out->end = out->buf + 4085; // 1 less than end plus extra for possible errors
+void
+oj_dump_obj_to_json(VALUE obj, Options copts, Out out) {
+ if (0 == out->buf) {
+ out->buf = ALLOC_N(char, 4096);
+ out->end = out->buf + 4085; // 1 less than end plus extra for possible errors
+ out->allocated = 1;
+ }
out->cur = out->buf;
out->circ_cnt = 0;
out->opts = copts;
@@ -1668,22 +1663,17 @@ dump_obj_to_json(VALUE obj, Options copts, Out out) {
}
}
-char*
-oj_write_obj_to_str(VALUE obj, Options copts) {
- struct _Out out;
-
- dump_obj_to_json(obj, copts, &out);
-
- return out.buf;
-}
-
void
oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
+ char buf[4096];
struct _Out out;
size_t size;
FILE *f;
- dump_obj_to_json(obj, copts, &out);
+ out.buf = buf;
+ out.end = buf + sizeof(buf) - 10;
+ out.allocated = 0;
+ oj_dump_obj_to_json(obj, copts, &out);
size = out.cur - out.buf;
if (0 == (f = fopen(path, "w"))) {
rb_raise(rb_eIOError, "%s\n", strerror(errno));
@@ -1693,7 +1683,9 @@ oj_write_obj_to_file(VALUE obj, const char *path, Options copts) {
rb_raise(rb_eIOError, "Write failed. [%d:%s]\n", err, strerror(err));
}
- xfree(out.buf);
+ if (out.allocated) {
+ xfree(out.buf);
+ }
fclose(f);
}
@@ -1872,10 +1864,13 @@ dump_leaf(Leaf leaf, int depth, Out out) {
}
}
-static void
-dump_leaf_to_json(Leaf leaf, Options copts, Out out) {
- out->buf = ALLOC_N(char, 4096);
- out->end = out->buf + 4085; // 10 less than end plus extra for possible errors
+void
+oj_dump_leaf_to_json(Leaf leaf, Options copts, Out out) {
+ if (0 == out->buf) {
+ out->buf = ALLOC_N(char, 4096);
+ out->end = out->buf + 4085; // 1 less than end plus extra for possible errors
+ out->allocated = 1;
+ }
out->cur = out->buf;
out->circ_cnt = 0;
out->opts = copts;
@@ -1884,22 +1879,17 @@ dump_leaf_to_json(Leaf leaf, Options copts, Out out) {
dump_leaf(leaf, 0, out);
}
-char*
-oj_write_leaf_to_str(Leaf leaf, Options copts) {
- struct _Out out;
-
- dump_leaf_to_json(leaf, copts, &out);
-
- return out.buf;
-}
-
void
oj_write_leaf_to_file(Leaf leaf, const char *path, Options copts) {
+ char buf[4096];
struct _Out out;
size_t size;
FILE *f;
- dump_leaf_to_json(leaf, copts, &out);
+ out.buf = buf;
+ out.end = buf + sizeof(buf) - 10;
+ out.allocated = 0;
+ oj_dump_leaf_to_json(leaf, copts, &out);
size = out.cur - out.buf;
if (0 == (f = fopen(path, "w"))) {
rb_raise(rb_eIOError, "%s\n", strerror(errno));
@@ -1909,6 +1899,8 @@ oj_write_leaf_to_file(Leaf leaf, const char *path, Options copts) {
rb_raise(rb_eIOError, "Write failed. [%d:%s]\n", err, strerror(err));
}
- xfree(out.buf);
+ if (out.allocated) {
+ xfree(out.buf);
+ }
fclose(f);
}
View
15 ext/oj/fast.c
@@ -1571,13 +1571,20 @@ doc_dump(int argc, VALUE *argv, VALUE self) {
}
}
if (0 != (leaf = get_doc_leaf(doc, path))) {
- char *json;
VALUE rjson;
if (0 == filename) {
- json = oj_write_leaf_to_str(leaf, &oj_default_options);
- rjson = rb_str_new2(json);
- xfree(json);
+ char buf[4096];
+ struct _Out out;
+
+ out.buf = buf;
+ out.end = buf + sizeof(buf) - 10;
+ out.allocated = 0;
+ oj_dump_leaf_to_json(leaf, &oj_default_options, &out);
+ rjson = rb_str_new2(out.buf);
+ if (out.allocated) {
+ xfree(out.buf);
+ }
} else {
oj_write_leaf_to_file(leaf, filename, &oj_default_options);
rjson = Qnil;
View
50 ext/oj/oj.c
@@ -573,22 +573,28 @@ strict_load(VALUE self, VALUE doc) {
*/
static VALUE
dump(int argc, VALUE *argv, VALUE self) {
- char *json;
+ char buf[4096];
+ struct _Out out;
struct _Options copts = oj_default_options;
VALUE rstr;
if (2 == argc) {
parse_options(argv[1], &copts);
}
- if (0 == (json = oj_write_obj_to_str(*argv, &copts))) {
+ out.buf = buf;
+ out.end = buf + sizeof(buf) - 10;
+ out.allocated = 0;
+ oj_dump_obj_to_json(*argv, &copts, &out);
+ if (0 == out.buf) {
rb_raise(rb_eNoMemError, "Not enough memory.");
}
- rstr = rb_str_new2(json);
+ rstr = rb_str_new2(out.buf);
#if HAS_ENCODING_SUPPORT
rb_enc_associate(rstr, oj_utf8_encoding);
#endif
- xfree(json);
-
+ if (out.allocated) {
+ xfree(out.buf);
+ }
return rstr;
}
@@ -698,14 +704,19 @@ saj_parse(int argc, VALUE *argv, VALUE self) {
static VALUE
mimic_dump(int argc, VALUE *argv, VALUE self) {
- char *json;
+ char buf[4096];
+ struct _Out out;
struct _Options copts = oj_default_options;
VALUE rstr;
- if (0 == (json = oj_write_obj_to_str(*argv, &copts))) {
+ out.buf = buf;
+ out.end = buf + sizeof(buf) - 10;
+ out.allocated = 0;
+ oj_dump_obj_to_json(*argv, &copts, &out);
+ if (0 == out.buf) {
rb_raise(rb_eNoMemError, "Not enough memory.");
}
- rstr = rb_str_new2(json);
+ rstr = rb_str_new2(out.buf);
#if HAS_ENCODING_SUPPORT
rb_enc_associate(rstr, oj_utf8_encoding);
#endif
@@ -717,8 +728,9 @@ mimic_dump(int argc, VALUE *argv, VALUE self) {
rb_funcall2(io, oj_write_id, 1, args);
rstr = io;
}
- xfree(json);
-
+ if (out.allocated) {
+ xfree(out.buf);
+ }
return rstr;
}
@@ -786,9 +798,13 @@ mimic_dump_load(int argc, VALUE *argv, VALUE self) {
static VALUE
mimic_generate_core(int argc, VALUE *argv, Options copts) {
- char *json;
- VALUE rstr;
+ char buf[4096];
+ struct _Out out;
+ VALUE rstr;
+ out.buf = buf;
+ out.end = buf + sizeof(buf) - 10;
+ out.allocated = 0;
if (2 == argc && Qnil != argv[1]) {
struct _DumpOpts dump_opts;
VALUE ropts = argv[1];
@@ -841,15 +857,17 @@ mimic_generate_core(int argc, VALUE *argv, Options copts) {
// :allow_nan is not supported as Oj always allows_nan
// :max_nesting is always set to 100
}
- if (0 == (json = oj_write_obj_to_str(*argv, copts))) {
+ oj_dump_obj_to_json(*argv, copts, &out);
+ if (0 == out.buf) {
rb_raise(rb_eNoMemError, "Not enough memory.");
}
- rstr = rb_str_new2(json);
+ rstr = rb_str_new2(out.buf);
#if HAS_ENCODING_SUPPORT
rb_enc_associate(rstr, oj_utf8_encoding);
#endif
- xfree(json);
-
+ if (out.allocated) {
+ xfree(out.buf);
+ }
return rstr;
}
View
18 ext/oj/oj.h
@@ -50,6 +50,7 @@ extern "C" {
#include <pthread.h>
#endif
#include "cache.h"
+#include "cache8.h"
#ifdef RUBINIUS_RUBY
#undef T_RATIONAL
@@ -116,6 +117,19 @@ typedef struct _Options {
DumpOpts dump_opts;
} *Options;
+typedef struct _Out {
+ char *buf;
+ char *end;
+ char *cur;
+ Cache8 circ_cache;
+ slot_t circ_cnt;
+ int indent;
+ int depth; // used by dump_hash
+ Options opts;
+ uint32_t hash_cnt;
+ int allocated;
+} *Out;
+
typedef struct _Odd {
VALUE clas; // Ruby class
VALUE create_obj;
@@ -149,9 +163,9 @@ typedef struct _Leaf {
extern VALUE oj_parse(char *json, Options options);
extern void oj_saj_parse(VALUE handler, char *json);
-extern char* oj_write_obj_to_str(VALUE obj, Options copts);
+extern void oj_dump_obj_to_json(VALUE obj, Options copts, Out out);
extern void oj_write_obj_to_file(VALUE obj, const char *path, Options copts);
-extern char* oj_write_leaf_to_str(Leaf leaf, Options copts);
+extern void oj_dump_leaf_to_json(Leaf leaf, Options copts, Out out);
extern void oj_write_leaf_to_file(Leaf leaf, const char *path, Options copts);
extern void _oj_raise_error(const char *msg, const char *xml, const char *current, const char* file, int line);
View
2  lib/oj/version.rb
@@ -1,5 +1,5 @@
module Oj
# Current version of the module.
- VERSION = '2.0.10.a1'
+ VERSION = '2.0.10.a2'
end
Please sign in to comment.
Something went wrong with that request. Please try again.