Skip to content
Permalink
Browse files

- We try to recycle array/table keeping track of the map of composite…

…-objects' id

  and their instance. It improve the performance.

- Add more testing cases.
  • Loading branch information...
Shuxin Yang
Shuxin Yang committed Dec 3, 2014
1 parent 0120e70 commit 7d2ce8f5f64ae02f8f30041c7d3300679d1027d0
Showing with 167 additions and 26 deletions.
  1. +2 −2 bench.lua
  2. +82 −21 json_decoder.lua
  3. +11 −3 tests/test.lua
  4. +72 −0 tests/test_cmp.lua
@@ -4,12 +4,12 @@ local f, err = io.open("bench.json", "r")

local iter = 100000
--local iter = 3
local instance = ljson_decoder.create()
local instance = ljson_decoder.new()

for line in f:lines() do
local begin = os.clock()
for i = 1, iter do
local result, err = ljson_decoder.decode(instance, line)
local result, err = instance:decode(line)
end

local t1 = os.clock() - begin
@@ -64,7 +64,10 @@ local ffi_cast = ffi.cast
local ffi_string = ffi.string

local _M = {}
local tab_new = require 'table.new'
local ok, tab_new = pcall(require, "table.new")
if not ok then
tab_new = function (narr, nrec) return {} end
end

local jp_lib = nil
local jp_create = nil
@@ -93,7 +96,7 @@ local function find_shared_obj(cpath, so_name)
end
end

function _M.load_json_parser()
function load_json_parser()
if jp_lib ~= nil then
return jp_lib
else
@@ -110,20 +113,6 @@ function _M.load_json_parser()
end

function _M.create()
if not jp_lib then
_M.load_json_parser()
end

if not jp_lib then
return nil, "fail to load libjson.so"
end

local parser_inst = jp_create()
if parser_inst ~= nil then
return ffi.gc(parser_inst, jp_destroy)
end

return nil, "Fail to create json paprser, likely due to OOM"
end

local ty_int64 = 0
@@ -226,7 +215,6 @@ end
convert_obj = function(obj, cobj_array)
local ty = obj.obj_ty
if ty <= ty_last_primitive then
--return nil
return create_primitive(ffi_cast(pobj_ptr_t, obj))
elseif ty == ty_array then
return create_array(ffi_cast(cobj_ptr_t, obj), cobj_array)
@@ -235,8 +223,73 @@ convert_obj = function(obj, cobj_array)
end
end

function _M.decode(parser_inst, json)
local objs = jp_parse(parser_inst, json, #json)
-- Create an array big enough to accommodate elmt_num + 2 elements.
-- If cobj_vect is big enough, return it; otherwise, create a new one.
local function create_cobj_vect(cobj_vect, elmt_num)
local array_size = elmt_num + 2
local cap = cobj_vect[0]
if cap < 400 and cap >= array_size then
return cobj_vect
end

cobj_vect = tab_new(array_size, 1)
cobj_vect[0] = array_size
return cobj_vect
end

-- set each element to be nil, such that they can be GC-ed ASAP.
local function clean_cobj_vect(cobj_vect, elmt_num)
for iter = 1, elmt_num + 2 do
cobj_vect[iter] = nil
end
end

-- #########################################################################
--
-- "Export" functions
--
-- #########################################################################
local setmetatable = setmetatable
local mt = { __index = _M }

function _M.new()
if not jp_lib then
load_json_parser()
end

if not jp_lib then
return nil, "fail to load libjson.so"
end

local parser_inst = jp_create()
if parser_inst ~= nil then
ffi.gc(parser_inst, jp_destroy)
else
return nil, "Fail to create JSON parser, likely due to OOM"
end

local cobj_vect = tab_new(100, 1)
if cobj_vect then
cobj_vect[0] = 100
else
return nil, "fail to create intermediate array"
end

local self = {
cobj_vect = cobj_vect,
parser = parser_inst
}

return setmetatable(self, mt)
end

function _M.decode(self, json)
--[[
if not self then
return nil, "JSON parser was not initialized properly"
end]]

local objs = jp_parse(self.parser, json, #json)
if objs == nil then
return nil, ffi.string(jp_get_err(parser_inst))
end
@@ -247,17 +300,26 @@ function _M.decode(parser_inst, json)
end

local composite_objs = ffi_cast(cobj_ptr_t, objs)
local cobj_vect = tab_new(composite_objs.id + 2, 0)
local elmt_num = composite_objs.id
local cobj_vect = create_cobj_vect(self.cobj_vect, elmt_num)
self.cobj_vect = cobj_vect

local last_val = nil
repeat
last_val = convert_obj(ffi_cast(obj_ptr_t, composite_objs), cobj_vect)
composite_objs = composite_objs.reverse_nesting_order
until composite_objs == nil

clean_cobj_vect(cobj_vect, elmt_num)

return last_val
end

-- #########################################################################
--
-- Debugging and Misc
--
-- #########################################################################
local print_primitive
local print_table
local print_var
@@ -285,7 +347,6 @@ print_table = function(array)
end
io.write("}");
end

print_var = function(var)
if type(var) == "table" then
print_table(var)
@@ -2,7 +2,7 @@ package.cpath = package.cpath..";../?.so"
package.path = package.cpath..";../?.lua"

local ljson_decoder = require 'json_decoder'
local f, err = io.open("../t2.json", "r")
local decoder = ljson_decoder.new()

local function cmp_lua_var(obj1, obj2)
if type(obj1) ~= type(obj2) then
@@ -11,7 +11,7 @@ local function cmp_lua_var(obj1, obj2)

if type(obj1) == "string" or
type(obj1) == "number" or
type(obj1) == "ni" or
type(obj1) == "nil" or
type(obj1) == "boolean" then
return obj1 == obj2 and true or nil
end
@@ -45,7 +45,7 @@ local test_total = 0;
local function ljson_test(test_id, parser, input, expect)
test_total = test_total + 1
io.write(string.format("Testing %s ...", test_id))
local result = ljson_decoder.decode(parser, input)
local result = decoder:decode(input)
if cmp_lua_var(result, expect) then
print("succ!")
else
@@ -72,6 +72,14 @@ input = [=[[{}]]=]
output = {{}}
ljson_test("test3", json_parser, input, output);

input = [=[[null]]=]
output = {nil}
ljson_test("test4", json_parser, input, output);

input = [=[[true, false]]=]
output = {true, false}
ljson_test("test5", json_parser, input, output);

io.write(string.format(
"\n============================\nTotal test count %d, fail %d\n",
test_total, test_fail_num))
@@ -0,0 +1,72 @@
package.cpath = package.cpath .. ";../?.so"
package.path = package.path .. ";../?.lua"

local cjson = require "cjson"
local ljson_decoder = require 'json_decoder'

local f, err = io.open("test_cmp.json", "r")

local function cmp_lua_var(obj1, obj2)
if type(obj1) ~= type(obj2) then
return
end

if type(obj1) == "string" or
type(obj1) == "number" or
type(obj1) == "nil" or
type(obj1) == "boolean" then
if obj1 == obj2 then
return true
end

print(obj1, "of tyep", type(obj1), "vs", obj2, "of type", obj2)
return
end

if (type(obj1) ~= "table") then
print("unknown type", type(obj1));
return
end

-- compare table
for k, v in pairs(obj1) do
if not cmp_lua_var(v, obj2[k]) then
print("key =", k, "value:", v," vs ", obj2[k])
return
end
end

for k, v in pairs(obj2) do
if not cmp_lua_var(v, obj1[k]) then
print("key =", k, "value:", v," vs ", obj1[k])
return
end
end

return true
end

local instance, err = ljson_decoder.new()
if not instance then
print("fail to create decoder instance")
end

local linenum = 0
local fail_num = 0;
for line in f:lines() do
local result1 = instance:decode(line)
local result2 = cjson.decode(line)

linenum = linenum + 1

if not cmp_lua_var(result1, result2) then
print("Fail with JSON at line", linenum)
fail_num = fail_num + 1
end
end

if fail_num == 0 then
print("pass!")
else
print("Fail!")
end

0 comments on commit 7d2ce8f

Please sign in to comment.
You can’t perform that action at this time.