optimize json encoding to use one nif call perf encoding instead of many

c_src/encode.c

@@ -29,73 +29,93 @@ fill_buffer(void* vctx,
             return;
         }
     }
-    memcpy(ctx->bin.data+ctx->fill_offset,str,len);
+    memcpy(ctx->bin.data + ctx->fill_offset, str, len);
     ctx->fill_offset += len;
 }
 
-ERL_NIF_TERM
-encode_string(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+
+static int
+encode_string(ErlNifEnv* env, ERL_NIF_TERM binary, ERL_NIF_TERM *pterm)
 {
     encode_ctx ctx;
-    ERL_NIF_TERM ret;
     ErlNifBinary bin;
 
-    if(!enif_inspect_iolist_as_binary(env, argv[0], &bin))
+    if(!enif_inspect_binary(env, binary, &bin))
     {
-        ret = enif_make_badarg(env);
-        goto done;
+        return 0;
     }
     if (!enif_alloc_binary(bin.size + 2, &(ctx.bin)))
     {
-        ret =  enif_make_tuple(env, 2,
-            enif_make_atom(env, "error"),
-            enif_make_atom(env, "insufficient_memory")
-        );
-        goto done;
+        return 0;
     }
     ctx.env = env;
     ctx.fill_offset = 0;
     ctx.fatal_error = 0;
-
-    fill_buffer(&ctx,"\"", 1);
+    fill_buffer(&ctx, "\"", 1);
     yajl_string_encode2(fill_buffer, &ctx, bin.data, bin.size);
-    fill_buffer(&ctx,"\"", 1);
+    fill_buffer(&ctx, "\"", 1);
 
     if (ctx.fatal_error) {
-        ret =  enif_make_tuple(env, 2,
-            enif_make_atom(env, "error"),
-            enif_make_atom(env, "insufficient_memory")
-            );
-    }
-    else {
-        ret = enif_make_binary(env, &ctx.bin);
+        return 0;
     }
-done:
-    return ret;
+     /* size might be bigger than the fill offset,
+     make smaller to truncate the garbage. */
+    ctx.bin.size = ctx.fill_offset;
+    *pterm = enif_make_binary(env, &ctx.bin);
+    return 1;
 }
 
-
 ERL_NIF_TERM
-encode_double(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+final_encode(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    ERL_NIF_TERM ret;
+#define BADARG ret = enif_make_badarg(env); goto done;
+
+    ERL_NIF_TERM ret = enif_make_list_from_array(env, NULL, 0);
+    ERL_NIF_TERM head = argv[0];
+    ERL_NIF_TERM term;
     double number;
     char buffer[32];
-
-    if(argc != 1 || !enif_get_double(env, argv[0], &number))
-    {
-        ret = enif_make_badarg(env);
-        goto done;
-    }
-    if (isnan(number) || isinf(number)) {
-        ret =  enif_make_tuple(env, 2,
-            enif_make_atom(env, "error"),
-            enif_make_atom(env, "invalid_number")
-        );
-    } else {
-        snprintf(buffer, sizeof(buffer), "%g", number);
-        ret = enif_make_string(env, buffer, ERL_NIF_LATIN1);
+    while(enif_get_list_cell(env, head, &term, &head)) {
+        //We scan the list, looking for things to encode. Most items we
+        // output untouched, but tuples we be tagged with a type and a 
+        //value: {Type, Value} where Type is a an Integer.
+
+        if (enif_is_tuple(env, term)) {
+            // It's a tuple.
+            const ERL_NIF_TERM* array;
+            int arity;
+            int code;
+            if (!enif_get_tuple(env, term, &arity, &array)) {
+                ret =  enif_make_tuple(env, 2,
+                    enif_make_atom(env, "error"),
+                    enif_make_atom(env, "invalid_number")
+                );
+            }
+            if (arity != 2 || !enif_get_int(env, array[0], &code)) {
+                BADARG;
+            }
+            if (code == 0) {
+                // {0, String}
+                if (!encode_string(env, array[1], &term)) {
+                    BADARG;
+                }
+            }
+            else {
+                // {1, Double}
+                if(!enif_get_double(env, array[1], &number)) {
+                    BADARG;
+                }
+                if (isnan(number) || isinf(number)) {
+                    BADARG;
+                }
+                snprintf(buffer, sizeof(buffer), "%.16g", number);
+                term = enif_make_string(env, buffer, ERL_NIF_LATIN1);
+            }
+        }
+        // Now encode the term into a new list head
+        ret = enif_make_list_cell(env, term, ret);
     }
 done:
     return ret;
 }
+

c_src/json.c

@@ -1,7 +1,6 @@
 #include "erl_nif.h"
 
-ERL_NIF_TERM encode_string(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
-ERL_NIF_TERM encode_double(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+ERL_NIF_TERM final_encode(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
 ERL_NIF_TERM reverse_tokens(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
 
 int
@@ -24,8 +23,7 @@ on_upgrade(ErlNifEnv* env, void** priv_data, void** old_data, ERL_NIF_TERM info)
 
 static ErlNifFunc nif_funcs[] =
 {
-    {"encode_string", 1, encode_string},
-    {"encode_double", 1, encode_double},
+    {"final_encode", 1, final_encode},
     {"reverse_tokens", 1, reverse_tokens}
 };
 

src/json.erl

@@ -20,59 +20,61 @@ decode(IoList) ->
     case reverse_tokens(IoList) of
     {ok, ReverseTokens} ->
         [[EJson]] = make_ejson(ReverseTokens, [[]]),
-        EJson;
+        {ok, EJson};
     Error ->
         Error
     end.
 
 
+encode(EJson) ->
+    try
+        RevList = encode_rev(EJson),
+        final_encode(lists:flatten(RevList))
+    catch throw:Error ->
+        Error
+    end.
 
-encode(true) ->
+% Encode the json into a reverse list that's almost an iolist
+% everything in the list is the final output except for tuples with
+% {0, Strings} and {1, Floats}, which are to be converted to strings
+% inside the NIF. The Nif also reverses the output, producing the final
+% iolist.
+encode_rev(true) ->
     <<"true">>;
-encode(false) ->
+encode_rev(false) ->
     <<"false">>;
-encode(null) ->
+encode_rev(null) ->
     <<"null">>;
-encode(I) when is_integer(I) ->
+encode_rev(I) when is_integer(I) ->
     integer_to_list(I);
-encode(F) when is_float(F) ->
-    encode_double(F);
-encode(S) when is_binary(S); is_atom(S) ->
-    encode_string(S);
-encode({Props}) when is_list(Props) ->
-    encode_proplist(Props);
-encode(Array) when is_list(Array) ->
-    encode_array(Array);
-encode(Bad) ->
-    exit({encode, {bad_term, Bad}}).
-
-encode_array([]) ->
-    <<"[]">>;
-encode_array(L) ->
-    F = fun (O, Acc) ->
-                [$,, encode(O) | Acc]
-        end,
-    [$, | Acc1] = lists:foldl(F, "[", L),
-    lists:reverse([$\] | Acc1]).
-
-encode_proplist([]) ->
-    <<"{}">>;
-encode_proplist(Props) ->
-    F = fun ({K, V}, Acc) ->
-                KS = encode_string(K),
-                VS = encode(V),
-                [$,, VS, $:, KS | Acc]
-        end,
-    [$, | Acc1] = lists:foldl(F, "{", Props),
-    lists:reverse([$\} | Acc1]).
-
-
-encode_string(_) ->
-    not_loaded(?LINE).
+encode_rev(S) when is_binary(S) ->
+    {0, S};
+encode_rev(F) when is_float(F) ->
+    {1, F};
+encode_rev({Props}) when is_list(Props) ->
+    encode_proplist_rev(Props, [<<"{">>]);
+encode_rev(Array) when is_list(Array) ->
+    encode_array_rev(Array, [<<"[">>]);
+encode_rev(Bad) ->
+    throw({encode, {bad_term, Bad}}).
+
+
+encode_array_rev([], Acc) ->
+    [<<"]">> | Acc];
+encode_array_rev([Val | Rest], [<<"[">>]) ->
+    encode_array_rev(Rest, [encode_rev(Val), <<"[">>]);
+encode_array_rev([Val | Rest], Acc) ->
+    encode_array_rev(Rest, [encode_rev(Val), <<",">> | Acc]).
+
+
+encode_proplist_rev([], Acc) ->
+    [<<"}">> | Acc];
+encode_proplist_rev([{Key,Val} | Rest], [<<"{">>]) ->
+    encode_proplist_rev(Rest, [{0, Key}, <<":">>, encode_rev(Val), <<"{">>]);
+encode_proplist_rev([{Key,Val} | Rest], Acc) ->
+    encode_proplist_rev(Rest, [{0, Key}, <<":">>, encode_rev(Val), <<",">> | Acc]).
 
 
-encode_double(_) ->
-    not_loaded(?LINE).
 
 
 make_ejson([], Stack) ->
@@ -108,6 +110,9 @@ fuzz(Chooser) ->
 
 not_loaded(Line) ->
     exit({json_not_loaded, module, ?MODULE, line, Line}).
-    
+
 reverse_tokens(_) ->
     not_loaded(?LINE).
+
+final_encode(_) ->
+    not_loaded(?LINE).