list 1

src/lib/yang/list.lua

@@ -2,9 +2,12 @@
 -- COPYING.
 module(..., package.seeall)
 
+local murmur = require("lib.hash.murmur")
 local ffi = require("ffi")
-local band, bor, lshift, rshift = bit.band, bit.bor, bit.lshift, bit.rshift
-local min, max = math.min, math.max
+local band, bor, bnot, lshift, rshift =
+   bit.band, bit.bor, bit.bnot, bit.lshift, bit.rshift
+local min, max =
+   math.min, math.max
 
 local Heap = {
    line_size = 128,
@@ -144,6 +147,8 @@ function Heap:allocate (bytes)
           or self:_bump_alloc(bytes)
    if o then
       self:_ref(o, bytes, 1)
+      -- Allocated space is zeroed. We are civilized, after all.
+      ffi.fill(self:ptr(o), bytes, 0)
       return o
    else
       self:_collect()
@@ -161,7 +166,7 @@ function Heap:ptr (o)
    return self._blocks[block].mem + offset
 end
 
-function selftest ()
+local function selftest_heap ()
    local h = Heap:new()
    local o1 = h:allocate(Heap.line_size/2)
    assert(h:_has_ref(0*Heap.line_size))
@@ -185,4 +190,338 @@ function selftest ()
    assert(h._maxfree == Heap.block_size*3)
    local o2 = h:allocate(Heap.line_size/2)
    local o3 = h:allocate(Heap.line_size)
+end
+
+
+List = {
+   trie_width = 4,
+   hash_width = 32,
+   node_entries = 16
+}
+
+List.type_map = {
+   decimal64 = 'double',
+   boolean = 'bool',
+   uint16 = 'uint16_t',
+   uint32 = 'uint32_t',
+   uint64 = 'uint64_t',
+   string = 'uint32_t'
+}
+
+List.type_cache = {}
+
+function List:cached_type (t)
+   if not self.type_cache[t] then
+      self.type_cache[t] = ffi.typeof(t)
+   end
+   return self.type_cache[t]
+end
+
+List.node_t = List:cached_type [[
+   struct {
+      uint16_t occupied, leaf;
+      uint32_t children[16];
+   }
+]]
+
+List.list_ts = [[
+   struct {
+      uint32_t prev, next;
+   }
+]]
+
+List.string_t = List:cached_type [[
+   struct {
+      uint16_t len;
+      uint8_t str[1];
+   } __attribute__((packed))
+]]
+
+function List:new (keys, members)
+   local self = setmetatable({}, {__index=List})
+   local keys_ts = self:build_type(keys)
+   local members_ts = self:build_type(members)
+   self.keys = keys
+   self.members = members
+   self.keys_t = self:cached_type(keys_ts)
+   self.leaf_t = self:cached_type(self:build_leaf_type(keys_ts, members_ts))
+   self.heap = Heap:new()
+   self.first, self.last = nil, nil -- empty
+   self.root = self:alloc_node() -- heap obj=0 reserved for root node
+   self.hashin = ffi.new(self.keys_t)
+   return self
+end
+
+function List:build_type (fields)
+   local t = "struct { "
+   for _, spec in ipairs(fields) do
+      t = t..("%s %s; "):format(
+         assert(self.type_map[spec[2]], "NYI: "..spec[2]),
+         spec[1]
+      )
+   end
+   t = t.."} __attribute__((packed))"
+   return t
+end
+
+function List:build_leaf_type (keys_ts, members_ts)
+   return ("struct { %s list; %s keys; %s members; } __attribute__((packed))")
+      :format(self.list_ts, keys_ts, members_ts)
+end
+
+local function ptrcast (t, ptr)
+   return ffi.cast(ffi.typeof('$*', t), ptr)
+end
+
+function List:alloc_node ()
+   local o = self.heap:allocate(ffi.sizeof(self.node_t))
+   return o
+end
+
+function List:free_node (o)
+   self.heap:free(o, ffi.sizeof(self.node_t))
+end
+
+function List:node (o)
+   return ptrcast(self.node_t, self.heap:ptr(o))
+end
+
+function List:alloc_leaf ()
+   local o = self.heap:allocate(ffi.sizeof(self.leaf_t))
+   return o
+end
+
+function List:free_leaf (o)
+   self.heap:free(o, ffi.sizeof(self.leaf_t))
+end
+
+function List:leaf (o)
+   return ptrcast(self.leaf_t, self.heap:ptr(o))
+end
+
+function List:alloc_str (s)
+   local o = self.heap:allocate(ffi.sizeof(self.string_t)+#s-1)
+   local str = ptrcast(self.string_t, self.heap:ptr(o))
+   ffi.copy(str.str, s, #s)
+   str.len = #s
+   return o
+end
+
+function List:free_str (o)
+   local str = ptrcast(self.string_t, self.heap:ptr(o))
+   self.heap:free(o, ffi.sizeof(self.string_t)+str.len-1)
+end
+
+function List:str (o)
+   return ptrcast(self.string_t, self.heap:ptr(o))
+end
+
+function List:tostring(str)
+   return ffi.string(str.str, str.len)
+end
+
+function List:copy_scalar (dst, src, fields)
+   for _, spec in ipairs(fields) do
+      local name, type = unpack(spec)
+      if type ~= 'string' then
+         dst[name] = src[name]
+      end
+   end
+end
+
+function List:totable (t, s, fields)
+   self:copy_scalar(t, s, fields)
+   for _, spec in ipairs(fields) do
+      local name, type = unpack(spec)
+      if type == 'string' then
+         t[name] = self:tostring(self:str(s[name]))
+      end
+   end
+end
+
+function List:tostruct (s, t, fields)
+   self:copy_scalar(s, t, fields)
+   for _, spec in ipairs(fields) do
+      local name, type = unpack(spec)
+      if type == 'string' then
+         s[name] = self:alloc_str(t[name])
+      end
+   end
+end
+
+local murmur32 = murmur.MurmurHash3_x86_32:new()
+local function hash32 (ptr, len, seed)
+   return murmur32:hash(ptr, len, seed).u32[0]
+end
+
+function List:entry_hash (e, seed)
+   self:copy_scalar(self.hashin, e, self.keys)
+   for _, spec in ipairs(self.keys) do
+      local name, type = unpack(spec)
+      if type == 'string' then
+         self.hashin[name] = hash32(e[name], #e[name], seed)
+      end
+   end
+   return hash32(self.hashin, ffi.sizeof(self.keys_t), seed)
+end
+
+-- Same as entry hash but for keys_t
+function List:leaf_hash (keys, seed)
+   self:copy_scalar(self.hashin, keys, self.keys)
+   for _, spec in ipairs(self.keys) do
+      local name, type = unpack(spec)
+      if type == 'string' then
+         local str = self:str(keys[name])
+         self.hashin[name] = hash32(str.str, str.len, seed)
+      end
+   end
+   return hash32(self.hashin, ffi.sizeof(self.keys_t), seed)
+end
+
+function List:new_leaf (e, prev, next)
+   local o = self:alloc_leaf()
+   local leaf = self:leaf(o)
+   leaf.list.prev = prev or 0 --  NB: obj=0 is root node, can not be a leaf!
+   leaf.list.next = next or 0
+   self:tostruct(leaf.keys, e, self.keys)
+   self:tostruct(leaf.members, e, self.members)
+   return o
+end
+
+function List:node_occupied (node, index, newval)
+   if newval == true then
+      node.occupied = bor(node.occupied, lshift(1, index))
+   elseif newval == false then
+      node.occupied = bor(node.occupied, bnot(lshift(1, index)))
+   end
+   return band(1, rshift(node.occupied, index)) == 1
+end
+
+function List:node_leaf (node, index, newval)
+   if newval == true then
+      node.leaf = bor(node.leaf, lshift(1, index))
+   elseif newval == false then
+      node.leaf = bor(node.leaf, bnot(lshift(1, index)))
+   end
+   return band(1, rshift(node.leaf, index)) == 1
+end
+
+function List:next_hash_parameters (d, s, h)
+   if d + 4 < self.hash_width then
+      return d + 4, s, h
+   else
+      return 0, s + 1, nil
+   end
+end
+
+function List:insert_leaf (o, r, d, s, h)
+   r = r or self.root
+   d = d or 0
+   s = s or 0
+   h = h or self:leaf_hash(self:leaf(o).keys, s)
+   local node = self:node(r)
+   local index = band(self.node_entries-1, rshift(h, d))
+   if self:node_occupied(node, index) then
+      -- Child slot occupied, advance hash parameters
+      d, s, h = self:next_hash_parameters()
+      if self:node_leaf(node, index) then
+         -- Occupied by leaf, replace with node and insert
+         -- both existing and new leaves into new node.
+         local l = node.children[index]
+         local n = self:alloc_node()
+         node.children[index] = n
+         self:node_leaf(node, index, false)
+         self:insert_leaf(l, n, d, s, nil)
+         self:insert_leaf(o, n, d, s, h)
+      else
+         -- Occupied by node, insert into it.
+         self:insert_leaf(o, r, d, s, h)
+      end
+   else
+      -- Not occupied, insert leaf.
+      self:node_occupied(node, index, true)
+      self:node_leaf(node, index, true)
+      node.children[index] = o
+   end
+end
+
+function List:find_leaf (k, r, d, s, h)
+   r = r or self.root
+   d = d or 0
+   s = s or 0
+   h = h or self:entry_hash(k, s)
+   local node = self:node(r)
+   local index = band(self.node_entries-1, rshift(h, d))
+   if self:node_occupied(node, index) then
+      if self:node_leaf(node, index) then
+         -- Found!
+         return node.children[index]
+      else
+         -- Continue searching in child node.
+         d, s, h = self:next_hash_parameters()
+         self:find_leaf(k, node.children[index], d, s, h)
+      end
+   else
+      -- Not present!
+      return nil
+   end
+end
+
+function List:append_leaf (o, prev)
+   local leaf = self:leaf(o)
+   local pleaf = self:leaf(prev)
+   leaf.list.prev = prev
+   leaf.list.next = pleaf.list.next
+   pleaf.list.next = o
+end
+
+function List:add_entry (e)
+   local o = self:new_leaf(e)
+   self:insert_leaf(o)
+   if self.last then
+      self:append_leaf(o, self.last)
+   else
+      self.last = o
+   end
+end
+
+function List:find_entry (k)
+   local o = self:find_leaf(k)
+   if o then
+      local leaf = self:leaf(o)
+      local ret = {}
+      self:totable(ret, leaf.keys, self.keys)
+      self:totable(ret, leaf.members, self.members)
+      return ret
+   end
+end
+
+function selftest_list ()
+   local l = List:new(
+      {{'id', 'uint32'}, {'name', 'string'}},
+      {{'value', 'decimal64'}, {'description', 'string'}}
+   )
+   print("leaf_t", ffi.sizeof(l.leaf_t))
+   print("node_t", ffi.sizeof(l.node_t))
+   l:add_entry{
+      id=42, name="foobar",
+      value=3.14, description="PI"
+   }
+   local root = l:node(l.root)
+   print(l.root, root.occupied, root.leaf, root.children[12])
+   local e1 = l:find_entry{
+      id=42, name="foobar"
+   }
+   assert(e1)
+   for k,v in pairs(e1) do
+      print(k,v)
+   end
+end
+
+
+function selftest ()
+   print("Selftest: Heap")
+   selftest_heap()
+   print("Selftest: List")
+   selftest_list()
 end