various improvements and bugfixes

- simplify the storage engine, so that msgpack/unpack is factored out; now just deals in bytes
- fix actual storage of values in leafs (which was broken before)
- shorten the key names for space-savings
- use different structures for nodes and leafs (dense v sparse)
- index node pointers by index, and not by prefix

bitslice.go

@@ -1,10 +1,11 @@
 package merkleTree
 
 import (
+	"encoding/binary"
 	"math/big"
 )
 
-func byteslice(h []byte, numBytes int, level int) []byte {
+func byteslice(h []byte, numBytes int, level int) ([]byte, uint32) {
 	start := numBytes * level
 	end := numBytes * (level + 1)
 	if start > len(h) {
@@ -13,10 +14,16 @@ func byteslice(h []byte, numBytes int, level int) []byte {
 	if end > len(h) {
 		end = len(h)
 	}
-	return h[start:end]
+	ret := h[start:end]
+
+	// Also map the returned byteslice to 0,1,2....2^numBytes
+	var tmp [4]byte
+	copy(tmp[(4-len(ret)):], ret)
+	index := binary.BigEndian.Uint32(tmp[:])
+	return ret, index
 }
 
-func bitslice(h []byte, numBits int, level int) Prefix {
+func bitslice(h []byte, numBits int, level int) (Prefix, uint32) {
 	if numBits%8 == 0 {
 		return byteslice(h, numBits/8, level)
 	}
@@ -39,5 +46,14 @@ func bitslice(h []byte, numBits int, level int) Prefix {
 	padlen := (end+7-begin)/8 - len(ret)
 	pad := make([]byte, padlen)
 	ret = append(pad, ret...)
-	return ret
+
+	tmp := z.Int64()
+	var index uint32
+	if tmp >= 0 && tmp <= 0xffffffff {
+		index = uint32(tmp)
+	} else {
+		panic("should never happen")
+	}
+
+	return ret, index
 }

bitslice_test.go

@@ -18,18 +18,22 @@ func TestBitslice(t *testing.T) {
 		output  []byte
 		numBits int
 		level   int
+		index   uint32
 	}{
-		{data, []byte{0x8c, 0xee}, 16, 1},
-		{data, []byte{0xa6}, 8, 1},
-		{data, []byte{0x0a}, 4, 2},
-		{data, []byte{0x00, 0xe3}, 9, 3},
+		{data, []byte{0x8c, 0xee}, 16, 1, uint32(0x8cee)},
+		{data, []byte{0xa6}, 8, 1, uint32(0xa6)},
+		{data, []byte{0x0a}, 4, 2, uint32(0x0a)},
+		{data, []byte{0x00, 0xe3}, 9, 3, uint32(0xe3)},
 	}
 
 	for i, v := range testVectors {
-		computed := bitslice(v.input, v.numBits, v.level)
+		computed, index := bitslice(v.input, v.numBits, v.level)
 		if !bytes.Equal(v.output, computed) {
 			t.Fatalf("failure in vector %d: got %v; wanted %v", i, computed, v.output)
 		}
+		if index != v.index {
+			t.Fatalf("wrong index: %d != %d", index, v.index)
+		}
 	}
 
 }

config.go

@@ -16,7 +16,7 @@ type Hasher interface {
 type ValueConstructor interface {
 	// Construct a new template empty value for the leaf, so that the
 	// Unmarshalling routine has the correct type template.
- 	Construct() interface{}
+	Construct() interface{}
 }
 
 // Config defines the shape of the MerkleTree.
@@ -52,11 +52,16 @@ func log2(y ChildIndex) ChildIndex {
 // and `n`, the maximum number of entries in a leaf before a
 // new level of the tree is introduced.
 func NewConfig(h Hasher, m ChildIndex, n ChildIndex, v ValueConstructor) Config {
-	return Config{hasher: h, m: m, n: n, c: log2(m), v : v}
+	return Config{hasher: h, m: m, n: n, c: log2(m), v: v}
 }
 
+func (c Config) prefixAndIndexAtLevel(level Level, h Hash) (Prefix, ChildIndex) {
+	prfx, ci := bitslice(h, int(c.c), int(level))
+	return Prefix(prfx), ChildIndex(ci)
+}
 func (c Config) prefixAtLevel(level Level, h Hash) Prefix {
-	return Prefix(bitslice(h, int(c.c), int(level)))
+	ret, _ := c.prefixAndIndexAtLevel(level, h)
+	return ret
 }
 
 func div8roundUp(i ChildIndex) ChildIndex {

errors.go

@@ -1,6 +1,7 @@
 package merkleTree
 
 import (
+	"errors"
 	"fmt"
 )
 
@@ -32,3 +33,6 @@ type BadChildPointerError struct {
 func (b BadChildPointerError) Error() string {
 	return fmt.Sprintf("Wanted a Hash; got type %T instead", b.V)
 }
+
+// ErrBadINode is thrown when we find a corrupt/malformed iNode
+var ErrBadINode = errors.New("Bad iNode found")

inode.go

@@ -0,0 +1,48 @@
+package merkleTree
+
+import ()
+
+type childPointerMap struct {
+	tab []Hash
+}
+
+func newChildPointerMap(capacity ChildIndex) *childPointerMap {
+	return &childPointerMap{
+		tab: make([]Hash, capacity),
+	}
+}
+
+func newChildPointerMapFromNode(n *Node) *childPointerMap {
+	return &childPointerMap{
+		tab: n.INodes,
+	}
+}
+
+func (c *childPointerMap) exportToNode(h Hasher, prevRoot Hash, level Level) (hash Hash, node Node, objExported []byte, err error) {
+	node.Type = nodeTypeINode
+	node.INodes = c.tab
+	return node.export(h, prevRoot, level)
+}
+
+func (n Node) export(h Hasher, prevRoot Hash, level Level) (hash Hash, node Node, objExported []byte, err error) {
+	if prevRoot != nil && level == Level(0) {
+		n.PrevRoot = prevRoot
+	}
+	objExported, err = encodeToBytes(n)
+	if err == nil {
+		hash = h.Hash(objExported)
+	}
+	return hash, n, objExported, err
+}
+
+func (n *Node) findChildByIndex(i ChildIndex) (Hash, error) {
+	if n.INodes == nil || int(i) >= len(n.INodes) {
+		return nil, ErrBadINode
+	}
+	return n.INodes[i], nil
+}
+
+func (c *childPointerMap) set(i ChildIndex, h Hash) *childPointerMap {
+	c.tab[i] = h
+	return c
+}

interfaces.go

@@ -3,8 +3,8 @@ package merkleTree
 // StorageEngine specifies how to store and lookup merkle tree nodes
 // and roots.  You can use a DB like Dynamo or SQL to do this.
 type StorageEngine interface {
-	StoreNode(Hash, Node, []byte) error
+	StoreNode(Hash, []byte) error
 	CommitRoot(prev Hash, curr Hash, txinfo TxInfo) error
-	LookupNode(Hash) (*Node, error)
+	LookupNode(Hash) []byte
 	LookupRoot() (Hash, error)
 }

mem.go

@@ -8,13 +8,13 @@ import (
 // MemEngine is an in-memory MerkleTree engine, used now mainly for testing
 type MemEngine struct {
 	root  Hash
-	nodes map[string](*Node)
+	nodes map[string][]byte
 }
 
 // NewMemEngine makes an in-memory storage engine, mainly for testing.
 func NewMemEngine() *MemEngine {
 	return &MemEngine{
-		nodes: make(map[string](*Node)),
+		nodes: make(map[string][]byte),
 	}
 }
 
@@ -33,9 +33,8 @@ func (m *MemEngine) Hash(d []byte) Hash {
 }
 
 // LookupNode looks up a MerkleTree node by hash
-func (m *MemEngine) LookupNode(h Hash) (*Node, error) {
-	ret := m.nodes[hex.EncodeToString(h)]
-	return ret, nil
+func (m *MemEngine) LookupNode(h Hash) (b []byte) {
+	return m.nodes[hex.EncodeToString(h)]
 }
 
 // LookupRoot fetches the root of the in-memory tree back out
@@ -44,7 +43,7 @@ func (m *MemEngine) LookupRoot() (Hash, error) {
 }
 
 // StoreNode stores the given node under the given key.
-func (m *MemEngine) StoreNode(key Hash, val Node, _ []byte) error {
-	m.nodes[hex.EncodeToString(key)] = &val
+func (m *MemEngine) StoreNode(key Hash, b []byte) error {
+	m.nodes[hex.EncodeToString(key)] = b
 	return nil
 }

obj_factory_test.go

@@ -17,8 +17,10 @@ func genBinary(l int) []byte {
 }
 
 type testValue struct {
-	seqno int
-	key   string
+	_struct bool `codec:",toarray"`
+	Seqno   int
+	Key     string
+	KeyRaw  []byte
 }
 
 // TestObjFactory generates a bunch of test objects for debugging
@@ -46,7 +48,7 @@ func (of TestObjFactory) dumpAll() []KeyValuePair {
 func (of *TestObjFactory) Produce() KeyValuePair {
 	key := genBinary(8)
 	keyString := hex.EncodeToString(key)
-	val := testValue{seqno: of.seqno, key: keyString}
+	val := testValue{Seqno: of.seqno, Key: keyString, KeyRaw: key}
 	of.seqno++
 	kvp := KeyValuePair{Key: key, Value: val}
 	of.objs[keyString] = kvp
@@ -70,8 +72,8 @@ func (of *TestObjFactory) modifySome(mod int) {
 			panic(fmt.Sprintf("Got value of wrong type: %T", v))
 		}
 		if (i % mod) == 0 {
-			tv.seqno *= 2
-			tv.key += "-yo-dawg"
+			tv.Seqno *= 2
+			tv.Key += "-yo-dawg"
 		}
 		i++
 	}

sorted_map.go

@@ -22,7 +22,7 @@ func NewSortedMapFromSortedList(l []KeyValuePair) *SortedMap {
 }
 
 func newSortedMapFromNode(n *Node) *SortedMap {
-	return NewSortedMapFromSortedList(n.Tab)
+	return NewSortedMapFromSortedList(n.Leafs)
 }
 
 // NewSortedMapFromList makes a sorted map from an unsorted list
@@ -51,15 +51,10 @@ func (s *SortedMap) push(kp KeyValuePair) {
 	s.list = append(s.list, kp)
 }
 
-func (s *SortedMap) exportToNode(h Hasher, typ nodeType, prevRoot Hash, level Level) (hash Hash, node Node, objExported []byte, err error) {
-	if prevRoot != nil && level == Level(0) {
-		node.PrevRoot = prevRoot
-	}
-	node.Type = typ
-	node.Tab = s.list
-	objExported, err = encodeToBytes(node)
-	hash = h.Hash(objExported)
-	return hash, node, objExported, err
+func (s *SortedMap) exportToNode(h Hasher, prevRoot Hash, level Level) (hash Hash, node Node, objExported []byte, err error) {
+	node.Type = nodeTypeLeaf
+	node.Leafs = s.list
+	return node.export(h, prevRoot, level)
 }
 
 func (s *SortedMap) binarySearch(k Hash) (ret int, eq bool) {
@@ -130,15 +125,3 @@ func (n *Node) findValueInLeaf(h Hash) interface{} {
 	}
 	return kvp.Value
 }
-
-func (n *Node) findChildByPrefix(p Prefix) (Hash, error) {
-	kvp := newSortedMapFromNode(n).find(Hash(p))
-	if kvp == nil {
-		return nil, nil
-	}
-	b, ok := (kvp.Value).(Hash)
-	if !ok {
-		return nil, BadChildPointerError{kvp.Value}
-	}
-	return Hash(b), nil
-}