diff --git a/src/lib/yang/list.lua b/src/lib/yang/list.lua index 939f27931a..8008523f6b 100644 --- a/src/lib/yang/list.lua +++ b/src/lib/yang/list.lua @@ -2,9 +2,12 @@ -- COPYING. module(..., package.seeall) +local murmur = require("lib.hash.murmur") local ffi = require("ffi") -local band, bor, lshift, rshift = bit.band, bit.bor, bit.lshift, bit.rshift -local min, max = math.min, math.max +local band, bor, bnot, lshift, rshift = + bit.band, bit.bor, bit.bnot, bit.lshift, bit.rshift +local min, max = + math.min, math.max local Heap = { line_size = 128, @@ -144,6 +147,8 @@ function Heap:allocate (bytes) or self:_bump_alloc(bytes) if o then self:_ref(o, bytes, 1) + -- Allocated space is zeroed. We are civilized, after all. + ffi.fill(self:ptr(o), bytes, 0) return o else self:_collect() @@ -161,7 +166,7 @@ function Heap:ptr (o) return self._blocks[block].mem + offset end -function selftest () +local function selftest_heap () local h = Heap:new() local o1 = h:allocate(Heap.line_size/2) assert(h:_has_ref(0*Heap.line_size)) @@ -185,4 +190,338 @@ function selftest () assert(h._maxfree == Heap.block_size*3) local o2 = h:allocate(Heap.line_size/2) local o3 = h:allocate(Heap.line_size) +end + + +List = { + trie_width = 4, + hash_width = 32, + node_entries = 16 +} + +List.type_map = { + decimal64 = 'double', + boolean = 'bool', + uint16 = 'uint16_t', + uint32 = 'uint32_t', + uint64 = 'uint64_t', + string = 'uint32_t' +} + +List.type_cache = {} + +function List:cached_type (t) + if not self.type_cache[t] then + self.type_cache[t] = ffi.typeof(t) + end + return self.type_cache[t] +end + +List.node_t = List:cached_type [[ + struct { + uint16_t occupied, leaf; + uint32_t children[16]; + } +]] + +List.list_ts = [[ + struct { + uint32_t prev, next; + } +]] + +List.string_t = List:cached_type [[ + struct { + uint16_t len; + uint8_t str[1]; + } __attribute__((packed)) +]] + +function List:new (keys, members) + local self = setmetatable({}, {__index=List}) + local keys_ts = self:build_type(keys) + local members_ts = self:build_type(members) + self.keys = keys + self.members = members + self.keys_t = self:cached_type(keys_ts) + self.leaf_t = self:cached_type(self:build_leaf_type(keys_ts, members_ts)) + self.heap = Heap:new() + self.first, self.last = nil, nil -- empty + self.root = self:alloc_node() -- heap obj=0 reserved for root node + self.hashin = ffi.new(self.keys_t) + return self +end + +function List:build_type (fields) + local t = "struct { " + for _, spec in ipairs(fields) do + t = t..("%s %s; "):format( + assert(self.type_map[spec[2]], "NYI: "..spec[2]), + spec[1] + ) + end + t = t.."} __attribute__((packed))" + return t +end + +function List:build_leaf_type (keys_ts, members_ts) + return ("struct { %s list; %s keys; %s members; } __attribute__((packed))") + :format(self.list_ts, keys_ts, members_ts) +end + +local function ptrcast (t, ptr) + return ffi.cast(ffi.typeof('$*', t), ptr) +end + +function List:alloc_node () + local o = self.heap:allocate(ffi.sizeof(self.node_t)) + return o +end + +function List:free_node (o) + self.heap:free(o, ffi.sizeof(self.node_t)) +end + +function List:node (o) + return ptrcast(self.node_t, self.heap:ptr(o)) +end + +function List:alloc_leaf () + local o = self.heap:allocate(ffi.sizeof(self.leaf_t)) + return o +end + +function List:free_leaf (o) + self.heap:free(o, ffi.sizeof(self.leaf_t)) +end + +function List:leaf (o) + return ptrcast(self.leaf_t, self.heap:ptr(o)) +end + +function List:alloc_str (s) + local o = self.heap:allocate(ffi.sizeof(self.string_t)+#s-1) + local str = ptrcast(self.string_t, self.heap:ptr(o)) + ffi.copy(str.str, s, #s) + str.len = #s + return o +end + +function List:free_str (o) + local str = ptrcast(self.string_t, self.heap:ptr(o)) + self.heap:free(o, ffi.sizeof(self.string_t)+str.len-1) +end + +function List:str (o) + return ptrcast(self.string_t, self.heap:ptr(o)) +end + +function List:tostring(str) + return ffi.string(str.str, str.len) +end + +function List:copy_scalar (dst, src, fields) + for _, spec in ipairs(fields) do + local name, type = unpack(spec) + if type ~= 'string' then + dst[name] = src[name] + end + end +end + +function List:totable (t, s, fields) + self:copy_scalar(t, s, fields) + for _, spec in ipairs(fields) do + local name, type = unpack(spec) + if type == 'string' then + t[name] = self:tostring(self:str(s[name])) + end + end +end + +function List:tostruct (s, t, fields) + self:copy_scalar(s, t, fields) + for _, spec in ipairs(fields) do + local name, type = unpack(spec) + if type == 'string' then + s[name] = self:alloc_str(t[name]) + end + end +end + +local murmur32 = murmur.MurmurHash3_x86_32:new() +local function hash32 (ptr, len, seed) + return murmur32:hash(ptr, len, seed).u32[0] +end + +function List:entry_hash (e, seed) + self:copy_scalar(self.hashin, e, self.keys) + for _, spec in ipairs(self.keys) do + local name, type = unpack(spec) + if type == 'string' then + self.hashin[name] = hash32(e[name], #e[name], seed) + end + end + return hash32(self.hashin, ffi.sizeof(self.keys_t), seed) +end + +-- Same as entry hash but for keys_t +function List:leaf_hash (keys, seed) + self:copy_scalar(self.hashin, keys, self.keys) + for _, spec in ipairs(self.keys) do + local name, type = unpack(spec) + if type == 'string' then + local str = self:str(keys[name]) + self.hashin[name] = hash32(str.str, str.len, seed) + end + end + return hash32(self.hashin, ffi.sizeof(self.keys_t), seed) +end + +function List:new_leaf (e, prev, next) + local o = self:alloc_leaf() + local leaf = self:leaf(o) + leaf.list.prev = prev or 0 -- NB: obj=0 is root node, can not be a leaf! + leaf.list.next = next or 0 + self:tostruct(leaf.keys, e, self.keys) + self:tostruct(leaf.members, e, self.members) + return o +end + +function List:node_occupied (node, index, newval) + if newval == true then + node.occupied = bor(node.occupied, lshift(1, index)) + elseif newval == false then + node.occupied = bor(node.occupied, bnot(lshift(1, index))) + end + return band(1, rshift(node.occupied, index)) == 1 +end + +function List:node_leaf (node, index, newval) + if newval == true then + node.leaf = bor(node.leaf, lshift(1, index)) + elseif newval == false then + node.leaf = bor(node.leaf, bnot(lshift(1, index))) + end + return band(1, rshift(node.leaf, index)) == 1 +end + +function List:next_hash_parameters (d, s, h) + if d + 4 < self.hash_width then + return d + 4, s, h + else + return 0, s + 1, nil + end +end + +function List:insert_leaf (o, r, d, s, h) + r = r or self.root + d = d or 0 + s = s or 0 + h = h or self:leaf_hash(self:leaf(o).keys, s) + local node = self:node(r) + local index = band(self.node_entries-1, rshift(h, d)) + if self:node_occupied(node, index) then + -- Child slot occupied, advance hash parameters + d, s, h = self:next_hash_parameters() + if self:node_leaf(node, index) then + -- Occupied by leaf, replace with node and insert + -- both existing and new leaves into new node. + local l = node.children[index] + local n = self:alloc_node() + node.children[index] = n + self:node_leaf(node, index, false) + self:insert_leaf(l, n, d, s, nil) + self:insert_leaf(o, n, d, s, h) + else + -- Occupied by node, insert into it. + self:insert_leaf(o, r, d, s, h) + end + else + -- Not occupied, insert leaf. + self:node_occupied(node, index, true) + self:node_leaf(node, index, true) + node.children[index] = o + end +end + +function List:find_leaf (k, r, d, s, h) + r = r or self.root + d = d or 0 + s = s or 0 + h = h or self:entry_hash(k, s) + local node = self:node(r) + local index = band(self.node_entries-1, rshift(h, d)) + if self:node_occupied(node, index) then + if self:node_leaf(node, index) then + -- Found! + return node.children[index] + else + -- Continue searching in child node. + d, s, h = self:next_hash_parameters() + self:find_leaf(k, node.children[index], d, s, h) + end + else + -- Not present! + return nil + end +end + +function List:append_leaf (o, prev) + local leaf = self:leaf(o) + local pleaf = self:leaf(prev) + leaf.list.prev = prev + leaf.list.next = pleaf.list.next + pleaf.list.next = o +end + +function List:add_entry (e) + local o = self:new_leaf(e) + self:insert_leaf(o) + if self.last then + self:append_leaf(o, self.last) + else + self.last = o + end +end + +function List:find_entry (k) + local o = self:find_leaf(k) + if o then + local leaf = self:leaf(o) + local ret = {} + self:totable(ret, leaf.keys, self.keys) + self:totable(ret, leaf.members, self.members) + return ret + end +end + +function selftest_list () + local l = List:new( + {{'id', 'uint32'}, {'name', 'string'}}, + {{'value', 'decimal64'}, {'description', 'string'}} + ) + print("leaf_t", ffi.sizeof(l.leaf_t)) + print("node_t", ffi.sizeof(l.node_t)) + l:add_entry{ + id=42, name="foobar", + value=3.14, description="PI" + } + local root = l:node(l.root) + print(l.root, root.occupied, root.leaf, root.children[12]) + local e1 = l:find_entry{ + id=42, name="foobar" + } + assert(e1) + for k,v in pairs(e1) do + print(k,v) + end +end + + +function selftest () + print("Selftest: Heap") + selftest_heap() + print("Selftest: List") + selftest_list() end \ No newline at end of file