feat: implement ViewDU.hashTreeRoot() using batch hash

packages/ssz/src/branchNodeStruct.ts

@@ -2,22 +2,27 @@ import {HashObject} from "@chainsafe/as-sha256/lib/hashObject";
 import {hashObjectToUint8Array, Node} from "@chainsafe/persistent-merkle-tree";
 
 /**
- * BranchNode whose children's data is represented as a struct, not a tree.
+ * BranchNode whose children's data is represented as a struct, the backed tree is lazily computed from the struct.
  *
  * This approach is usefull for memory efficiency of data that is not modified often, for example the validators
  * registry in Ethereum consensus `state.validators`. The tradeoff is that getting the hash, are proofs is more
  * expensive because the tree has to be recreated every time.
  */
 export class BranchNodeStruct<T> extends Node {
+  /**
+   * this represents the backed tree which is lazily computed from value
+   */
+  private _rootNode: Node | null = null;
   constructor(private readonly valueToNode: (value: T) => Node, readonly value: T) {
     // First null value is to save an extra variable to check if a node has a root or not
     super(null as unknown as number, 0, 0, 0, 0, 0, 0, 0);
+    this._rootNode = null;
   }
 
   get rootHashObject(): HashObject {
+    // return this.rootNode.rootHashObject;
     if (this.h0 === null) {
-      const node = this.valueToNode(this.value);
-      super.applyHash(node.rootHashObject);
+      super.applyHash(this.rootNode.rootHashObject);
     }
     return this;
   }
@@ -31,10 +36,21 @@ export class BranchNodeStruct<T> extends Node {
   }
 
   get left(): Node {
-    return this.valueToNode(this.value).left;
+    return this.rootNode.left;
   }
 
   get right(): Node {
-    return this.valueToNode(this.value).right;
+    return this.rootNode.right;
+  }
+
+  /**
+   * Singleton implementation to make sure there is single backed tree for this node.
+   * This is important for batching HashComputations
+   */
+  private get rootNode(): Node {
+    if (this._rootNode === null) {
+      this._rootNode = this.valueToNode(this.value);
+    }
+    return this._rootNode;
   }
 }

packages/ssz/src/type/arrayBasic.ts

@@ -1,10 +1,12 @@
 import {
   BranchNode,
+  HashComputationGroup,
   LeafNode,
   Node,
   getNodesAtDepth,
   packedNodeRootsToBytes,
   packedRootsBytesToNode,
+  arrayAtIndex,
 } from "@chainsafe/persistent-merkle-tree";
 import {Type, ValueOf, ByteViews} from "./abstract";
 import {BasicType} from "./basic";
@@ -39,14 +41,23 @@ export function addLengthNode(chunksNode: Node, length: number): Node {
   return new BranchNode(chunksNode, LeafNode.fromUint32(length));
 }
 
-export function setChunksNode(rootNode: Node, chunksNode: Node, newLength?: number): Node {
+export function setChunksNode(
+  rootNode: Node,
+  chunksNode: Node,
+  newLength: number | null,
+  hashComps: HashComputationGroup | null
+): Node {
   const lengthNode =
-    newLength !== undefined
+    newLength !== null
       ? // If newLength is set, create a new node for length
         LeafNode.fromUint32(newLength)
       : // else re-use existing node
         (rootNode.right as LeafNode);
-  return new BranchNode(chunksNode, lengthNode);
+  const branchNode = new BranchNode(chunksNode, lengthNode);
+  if (hashComps !== null) {
+    arrayAtIndex(hashComps.byLevel, hashComps.offset).push({src0: chunksNode, src1: lengthNode, dest: branchNode});
+  }
+  return branchNode;
 }
 
 export type ArrayProps = {isList: true; limit: number} | {isList: false; length: number};

packages/ssz/src/type/bitArray.ts

@@ -1,4 +1,4 @@
-import {concatGindices, Gindex, Node, toGindex, Tree} from "@chainsafe/persistent-merkle-tree";
+import {concatGindices, Gindex, HashComputationGroup, Node, toGindex, Tree} from "@chainsafe/persistent-merkle-tree";
 import {fromHexString, toHexString, byteArrayEquals} from "../util/byteArray";
 import {splitIntoRootChunks} from "../util/merkleize";
 import {CompositeType, LENGTH_GINDEX} from "./composite";
@@ -29,8 +29,8 @@ export abstract class BitArrayType extends CompositeType<BitArray, BitArrayTreeV
     return view.node;
   }
 
-  commitViewDU(view: BitArrayTreeViewDU): Node {
-    view.commit();
+  commitViewDU(view: BitArrayTreeViewDU, hashComps: HashComputationGroup | null = null): Node {
+    view.commit(hashComps);
     return view.node;
   }
 

packages/ssz/src/type/byteArray.ts

@@ -1,4 +1,4 @@
-import {concatGindices, Gindex, Node, toGindex, Tree} from "@chainsafe/persistent-merkle-tree";
+import {concatGindices, Gindex, HashComputationGroup, Node, toGindex, Tree} from "@chainsafe/persistent-merkle-tree";
 import {fromHexString, toHexString, byteArrayEquals} from "../util/byteArray";
 import {splitIntoRootChunks} from "../util/merkleize";
 import {ByteViews} from "./abstract";
@@ -37,7 +37,8 @@ export abstract class ByteArrayType extends CompositeType<ByteArray, ByteArray,
     return this.commitViewDU(view);
   }
 
-  commitViewDU(view: ByteArray): Node {
+  // TODO: batch
+  commitViewDU(view: ByteArray, hashComps: HashComputationGroup | null = null): Node {
     const uint8Array = new Uint8Array(this.value_serializedSize(view));
     const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength);
     this.value_serializeToBytes({uint8Array, dataView}, 0, view);

packages/ssz/src/type/composite.ts

@@ -3,6 +3,7 @@ import {
   createProof,
   getNode,
   Gindex,
+  HashComputationGroup,
   Node,
   Proof,
   ProofType,
@@ -126,7 +127,7 @@ export abstract class CompositeType<V, TV, TVDU> extends Type<V> {
   /** INTERNAL METHOD: Given a Tree View, returns a `Node` with all its updated data */
   abstract commitView(view: TV): Node;
   /** INTERNAL METHOD: Given a Deferred Update Tree View returns a `Node` with all its updated data */
-  abstract commitViewDU(view: TVDU): Node;
+  abstract commitViewDU(view: TVDU, hashComps?: HashComputationGroup | null): Node;
   /** INTERNAL METHOD: Return the cache of a Deferred Update Tree View. May return `undefined` if this ViewDU has no cache */
   abstract cacheOfViewDU(view: TVDU): unknown;
 

packages/ssz/src/type/container.ts

@@ -7,6 +7,7 @@ import {
   toGindex,
   concatGindices,
   getNode,
+  HashComputationGroup,
 } from "@chainsafe/persistent-merkle-tree";
 import {maxChunksToDepth} from "../util/merkleize";
 import {Require} from "../util/types";
@@ -162,8 +163,8 @@ export class ContainerType<Fields extends Record<string, Type<unknown>>> extends
     return view.node;
   }
 
-  commitViewDU(view: ContainerTreeViewDUType<Fields>): Node {
-    view.commit();
+  commitViewDU(view: ContainerTreeViewDUType<Fields>, hashComps: HashComputationGroup | null = null): Node {
+    view.commit(hashComps);
     return view.node;
   }
 

packages/ssz/src/type/listBasic.ts

@@ -1,4 +1,4 @@
-import {LeafNode, Node, Tree} from "@chainsafe/persistent-merkle-tree";
+import {HashComputationGroup, LeafNode, Node, Tree} from "@chainsafe/persistent-merkle-tree";
 import {ValueOf} from "./abstract";
 import {BasicType} from "./basic";
 import {ByteViews} from "./composite";
@@ -93,8 +93,8 @@ export class ListBasicType<ElementType extends BasicType<unknown>>
     return view.node;
   }
 
-  commitViewDU(view: ListBasicTreeViewDU<ElementType>): Node {
-    view.commit();
+  commitViewDU(view: ListBasicTreeViewDU<ElementType>, hashComps: HashComputationGroup | null = null): Node {
+    view.commit(hashComps);
     return view.node;
   }
 
@@ -144,8 +144,18 @@ export class ListBasicType<ElementType extends BasicType<unknown>>
     return node.left;
   }
 
-  tree_setChunksNode(rootNode: Node, chunksNode: Node, newLength?: number): Node {
-    return setChunksNode(rootNode, chunksNode, newLength);
+  tree_chunksNodeOffset(): number {
+    // one more level for length, see setChunksNode below
+    return 1;
+  }
+
+  tree_setChunksNode(
+    rootNode: Node,
+    chunksNode: Node,
+    newLength: number | null,
+    hashComps: HashComputationGroup | null
+  ): Node {
+    return setChunksNode(rootNode, chunksNode, newLength, hashComps);
   }
 
   // Merkleization

packages/ssz/src/type/listComposite.ts

@@ -1,4 +1,4 @@
-import {Node, Tree} from "@chainsafe/persistent-merkle-tree";
+import {HashComputationGroup, Node, Tree} from "@chainsafe/persistent-merkle-tree";
 import {
   mixInLength,
   maxChunksToDepth,
@@ -97,8 +97,8 @@ export class ListCompositeType<
     return view.node;
   }
 
-  commitViewDU(view: ListCompositeTreeViewDU<ElementType>): Node {
-    view.commit();
+  commitViewDU(view: ListCompositeTreeViewDU<ElementType>, hashComps: HashComputationGroup | null = null): Node {
+    view.commit(hashComps);
     return view.node;
   }
 
@@ -150,8 +150,18 @@ export class ListCompositeType<
     return node.left;
   }
 
-  tree_setChunksNode(rootNode: Node, chunksNode: Node, newLength?: number): Node {
-    return setChunksNode(rootNode, chunksNode, newLength);
+  tree_chunksNodeOffset(): number {
+    // one more level for length, see setChunksNode below
+    return 1;
+  }
+
+  tree_setChunksNode(
+    rootNode: Node,
+    chunksNode: Node,
+    newLength: number | null,
+    hashComps: HashComputationGroup | null
+  ): Node {
+    return setChunksNode(rootNode, chunksNode, newLength, hashComps);
   }
 
   // Merkleization

packages/ssz/src/type/optional.ts

@@ -1,4 +1,4 @@
-import {concatGindices, Gindex, Node, Tree, zeroNode} from "@chainsafe/persistent-merkle-tree";
+import {concatGindices, Gindex, HashComputationGroup, Node, Tree, zeroNode} from "@chainsafe/persistent-merkle-tree";
 import {mixInLength} from "../util/merkleize";
 import {Require} from "../util/types";
 import {namedClass} from "../util/named";
@@ -75,7 +75,8 @@ export class OptionalType<ElementType extends Type<unknown>> extends CompositeTy
   }
 
   // TODO add an OptionalViewDU
-  commitViewDU(view: ValueOfType<ElementType>): Node {
+  // TODO: batch
+  commitViewDU(view: ValueOfType<ElementType>, hashComps: HashComputationGroup | null = null): Node {
     return this.value_toTree(view);
   }
 

packages/ssz/src/type/union.ts

@@ -1,4 +1,4 @@
-import {concatGindices, getNode, Gindex, Node, Tree} from "@chainsafe/persistent-merkle-tree";
+import {concatGindices, getNode, Gindex, HashComputationGroup, Node, Tree} from "@chainsafe/persistent-merkle-tree";
 import {mixInLength} from "../util/merkleize";
 import {Require} from "../util/types";
 import {namedClass} from "../util/named";
@@ -106,7 +106,8 @@ export class UnionType<Types extends Type<unknown>[]> extends CompositeType<
     return this.value_toTree(view);
   }
 
-  commitViewDU(view: ValueOfTypes<Types>): Node {
+  // TODO: batch
+  commitViewDU(view: ValueOfTypes<Types>, hashComps: HashComputationGroup | null = null): Node {
     return this.value_toTree(view);
   }
 

packages/ssz/src/type/vectorBasic.ts

@@ -1,4 +1,4 @@
-import {Node, Tree} from "@chainsafe/persistent-merkle-tree";
+import {HashComputationGroup, Node, Tree} from "@chainsafe/persistent-merkle-tree";
 import {maxChunksToDepth, splitIntoRootChunks} from "../util/merkleize";
 import {Require} from "../util/types";
 import {namedClass} from "../util/named";
@@ -83,8 +83,8 @@ export class VectorBasicType<ElementType extends BasicType<unknown>>
     return view.node;
   }
 
-  commitViewDU(view: ArrayBasicTreeViewDU<ElementType>): Node {
-    view.commit();
+  commitViewDU(view: ArrayBasicTreeViewDU<ElementType>, hashComps: HashComputationGroup | null = null): Node {
+    view.commit(hashComps);
     return view.node;
   }
 
@@ -132,6 +132,10 @@ export class VectorBasicType<ElementType extends BasicType<unknown>>
     return node;
   }
 
+  tree_chunksNodeOffset(): number {
+    return 0;
+  }
+
   tree_setChunksNode(rootNode: Node, chunksNode: Node): Node {
     return chunksNode;
   }

packages/ssz/src/type/vectorComposite.ts

@@ -1,4 +1,4 @@
-import {Node, Tree} from "@chainsafe/persistent-merkle-tree";
+import {HashComputationGroup, Node, Tree} from "@chainsafe/persistent-merkle-tree";
 import {maxChunksToDepth} from "../util/merkleize";
 import {Require} from "../util/types";
 import {namedClass} from "../util/named";
@@ -90,8 +90,8 @@ export class VectorCompositeType<
     return view.node;
   }
 
-  commitViewDU(view: ArrayCompositeTreeViewDU<ElementType>): Node {
-    view.commit();
+  commitViewDU(view: ArrayCompositeTreeViewDU<ElementType>, hashComps: HashComputationGroup | null = null): Node {
+    view.commit(hashComps);
     return view.node;
   }
 
@@ -139,6 +139,10 @@ export class VectorCompositeType<
     return node;
   }
 
+  tree_chunksNodeOffset(): number {
+    return 0;
+  }
+
   tree_setChunksNode(rootNode: Node, chunksNode: Node): Node {
     return chunksNode;
   }

packages/ssz/src/view/arrayBasic.ts

@@ -1,4 +1,4 @@
-import {getNodesAtDepth, LeafNode, Node, Tree} from "@chainsafe/persistent-merkle-tree";
+import {getNodesAtDepth, HashComputationGroup, LeafNode, Node, Tree} from "@chainsafe/persistent-merkle-tree";
 import {ValueOf} from "../type/abstract";
 import {BasicType} from "../type/basic";
 import {CompositeType} from "../type/composite";
@@ -21,8 +21,15 @@ export type ArrayBasicType<ElementType extends BasicType<unknown>> = CompositeTy
   tree_setLength(tree: Tree, length: number): void;
   /** INTERNAL METHOD: Return the chunks node from a root node */
   tree_getChunksNode(rootNode: Node): Node;
+  /** INTERNAL METHOD: Return the offset from root for HashComputation */
+  tree_chunksNodeOffset(): number;
   /** INTERNAL METHOD: Return a new root node with changed chunks node and length */
-  tree_setChunksNode(rootNode: Node, chunksNode: Node, newLength?: number): Node;
+  tree_setChunksNode(
+    rootNode: Node,
+    chunksNode: Node,
+    newLength: number | null,
+    hashComps: HashComputationGroup | null
+  ): Node;
 };
 
 export class ArrayBasicTreeView<ElementType extends BasicType<unknown>> extends TreeView<ArrayBasicType<ElementType>> {

packages/ssz/src/view/arrayComposite.ts

@@ -1,4 +1,4 @@
-import {getNodesAtDepth, Node, toGindexBitstring, Tree} from "@chainsafe/persistent-merkle-tree";
+import {getNodesAtDepth, HashComputationGroup, Node, toGindexBitstring, Tree} from "@chainsafe/persistent-merkle-tree";
 import {ValueOf} from "../type/abstract";
 import {CompositeType, CompositeView, CompositeViewDU} from "../type/composite";
 import {TreeView} from "./abstract";
@@ -16,8 +16,15 @@ export type ArrayCompositeType<
   tree_setLength(tree: Tree, length: number): void;
   /** INTERNAL METHOD: Return the chunks node from a root node */
   tree_getChunksNode(rootNode: Node): Node;
+  /** INTERNAL METHOD: Return the offset from root for HashComputation */
+  tree_chunksNodeOffset(): number;
   /** INTERNAL METHOD: Return a new root node with changed chunks node and length */
-  tree_setChunksNode(rootNode: Node, chunksNode: Node, newLength?: number): Node;
+  tree_setChunksNode(
+    rootNode: Node,
+    chunksNode: Node,
+    newLength: number | null,
+    hashComps: HashComputationGroup | null
+  ): Node;
 };
 
 export class ArrayCompositeTreeView<