Merge pull request #5 from Knio/header_refactor

Header refactor

everdb/__init__.py

@@ -5,5 +5,6 @@
 from .database          import Database
 
 from .blob  import Blob
+from .page  import Page
 from .array import Array
 from .hash  import Hash

everdb/array.py

@@ -1,31 +1,36 @@
-import struct
+'''
+Array of single primitive (struct) items
+'''
+
+# pylint: disable=W0311
 
-from .blob import Blob
-from .blob import SMALL_BLOB, REGULAR_BLOB
-from .blob import BLOCK, OFFSET
-from .blob import ZERO_BLOCK, BLOCK_MASK, BLOCK_SIZE
+import struct
 
-class Array(Blob):
+from .page import Page, Field
+from .page import SMALL_PAGE, REGULAR_PAGE
+from .page import ZERO_BLOCK, BLOCK_MASK, BLOCK_SIZE
 
-  HEADER = dict((k, i) for i, k in enumerate([
-    'capacity',
-    'length',
-    'num_blocks',
-    'type',
-    'checksum', # must be last
-  ]))
+class Array(Page):
+  format    = Field('c')
+  length    = Field('Q')
 
   def __init__(self, host, root, format, new):
-    self.format = format
+    self.format = format.encode('ascii')
     self.item_size = struct.calcsize(format)
     self.items_per_block = BLOCK_SIZE // self.item_size
     super(Array, self).__init__(host, root, new)
 
   def init_root(self):
+    MAX_SMALL = (BLOCK_SIZE - self._header_size) // self.item_size
+    self.length = 0
     super(Array, self).init_root()
-    MAX_SMALL = (BLOCK_SIZE - 4 * len(Array.HEADER)) // self.item_size
-    self.capacity = MAX_SMALL
-    self.flush_root()
+
+  @property
+  def capacity(self):
+    if self.type == SMALL_PAGE:
+      MAX_SMALL = (BLOCK_SIZE - self._header_size)
+      return MAX_SMALL // self.item_size
+    return self.num_blocks * self.items_per_block
 
   def __len__(self):
     return self.length
@@ -43,13 +48,13 @@ def __getitem__(self, i):
     j = i // self.items_per_block
     k = i  % self.items_per_block
 
-    if self.type == SMALL_BLOB:
+    if self.type == SMALL_PAGE:
       b = self.host[self.root]
     else:
       b = self.get_block(j)
 
-    # TODO cache this
-    return b.cast(self.format)[k]
+    # TODO SLOW cache this
+    return b.cast(self.format.decode('ascii'))[k]
 
   def __setitem__(self, i, v):
     if isinstance(i, slice):
@@ -61,13 +66,13 @@ def __setitem__(self, i, v):
     j = i // self.items_per_block
     k = i  % self.items_per_block
 
-    if self.type == SMALL_BLOB:
+    if self.type == SMALL_PAGE:
       b = self.host[self.root]
     else:
       b = self.get_block(j)
 
-    # TODO cache this
-    b.cast(self.format)[k] = v
+    # TODO SLOW cache this
+    b.cast(self.format.decode('ascii'))[k] = v
 
   def getslice(self, i):
     raise NotImplementedError
@@ -85,87 +90,83 @@ def append(self, v):
     self[l] = v
     # ensure 1 extra slot
     if self.capacity < l + 2:
-      self.resize(l + 2)
+      if self.num_blocks == 0:
+        self.make_regular()
+      else:
+        self.allocate(self.num_blocks + 1)
 
   def pop(self):
     if not self.length:
       raise IndexError
     l = self.length
     j = l - 1
     x = self[j]
+    self[j] = 0
     self.length = j
-    if self.capacity > l + 2:
-      self.resize(l + 2)
+
+    if self.num_blocks == 1:
+      if (BLOCK_SIZE - self._header_size) // self.item_size > l:
+        self.make_small()
+
+    elif self.num_blocks > 1:
+      if (self.num_blocks - 1) * self.items_per_block > l:
+        self.allocate(self.num_blocks - 1)
+
     return x
 
-  def resize(self, capacity):
+  def resize(self, length):
     '''
     Resizes the array to a given size (in elements, not bytes)
     '''
     # requested size fits in a small block
     # handles both grow and shrink operation
-    length = capacity * self.item_size
-
-    MAX_SMALL = (BLOCK_SIZE - 4 * len(Array.HEADER))
+    MAX_SMALL = (BLOCK_SIZE - self._header_size) // self.item_size
     if length <= MAX_SMALL:
-      if self.type == REGULAR_BLOB:
+      s = slice(length, MAX_SMALL)
+      if self.type == REGULAR_PAGE:
         # copy data to root in small block
         data = bytes(self.get_block(0)[0:length])
         # free data + page blocks,
         self.allocate(0)
         self.host[self.root][0:length] = data
-      else:
-        # already a small block
-        # zero fill from requested length,
-        # to make sure truncated data is not leaked
-        s = slice(length, MAX_SMALL)
-        self.host[self.root][s] = ZERO_BLOCK[s]
-        # no zero fill when growing, since we assume the block was
-        # zeroed either above or on blob creation
-      self.type = SMALL_BLOB
-      self.capacity = capacity
+        self.type = SMALL_PAGE
+      # already a small block
+      # zero fill from requested length,
+      # to make sure truncated data is not leaked
+      self.host[self.root][s] = ZERO_BLOCK[s]
+      # no zero fill when growing, since we assume the block was
+      # zeroed either above or on blob creation
+      self.length = length
       self.flush_root()
       return
 
     # requested size requires a regular blob
-    num_blocks = BLOCK(length + BLOCK_MASK)
+    num_blocks = (length + self.items_per_block - 1) // self.items_per_block
     cur_blocks = self.num_blocks
 
     if cur_blocks == num_blocks:
       # don't need to allocate or free any blocks
       # zero fill any truncated space
-      i = BLOCK(length)
+      i = (length - 1) // self.items_per_block
       if i != cur_blocks:
         b = self.get_block(i)
-        s = slice(OFFSET(length), BLOCK_SIZE)
+        # TODO use length:self.length, not whole block
+        s = slice(length % self.items_per_block, BLOCK_SIZE)
         b[s] = ZERO_BLOCK[s]
-      self.capacity = capacity
+      self.length = length
       self.flush_root()
       return
 
-    data = None
-    l = self.length
-    if self.type == SMALL_BLOB:
-      data = bytes(self.host[self.root][0:self.length * self.item_size])
-      self.type = REGULAR_BLOB
-      self.length = 0
-      self.capacity = 0
-      # def _pop():
-      #   return data.pop()
-      # self.pop = _pop
+    if self.type == SMALL_PAGE:
+      self.make_regular()
 
     # allocate/free blocks
     # may call append/pop!
     self.allocate(num_blocks)
 
-
-    # copy data back from small block expansion
-    if data:
-      self.get_block(0)[0:len(data)] = data
-      self.length = l
-      # del self.pop
-
-    # still need this because we may have over-allocated
-    # if length was not on a page boundary
-    self.capacity = capacity
+    self.length = length
     self.flush_root()
+
+  def __repr__(self):
+    return '''<Array(root=%d, type=%d, format=%s, num_blocks=%d, length=%d)>''' % \
+        (self.root, self.type, self.format, self.num_blocks, self.length)