feat: implement ListCompositeTreeViewDU

packages/ssz/test/lodestarTypes/phase0/listValidator.ts

@@ -0,0 +1,15 @@
+import { ListCompositeType } from "../../../src/type/listComposite";
+import { Node } from "@chainsafe/persistent-merkle-tree";
+import { ListCompositeTreeViewDU } from "../../../src/viewDU/listComposite";
+import { ValidatorNodeStructType } from "./validator";
+import { ListValidatorTreeViewDU } from "./viewDU/listValidator";
+
+export class ListValidatorType extends ListCompositeType<ValidatorNodeStructType> {
+  constructor(limit: number) {
+    super(new ValidatorNodeStructType(), limit);
+  }
+
+  getViewDU(node: Node, cache?: unknown): ListCompositeTreeViewDU<ValidatorNodeStructType> {
+    return new ListValidatorTreeViewDU(this, node, cache as any);
+  }
+}

packages/ssz/test/lodestarTypes/phase0/sszTypes.ts

@@ -16,6 +16,7 @@ import {
   ATTESTATION_SUBNET_COUNT,
 } from "../params";
 import * as primitiveSsz from "../primitive/sszTypes";
+import {ListValidatorType} from "./listValidator";
 import {ValidatorNodeStruct} from "./validator";
 
 export {ValidatorNodeStruct};
@@ -251,7 +252,8 @@ export const ValidatorContainer = new ContainerType(
 export const Validator = ValidatorNodeStruct;
 
 // Export as stand-alone for direct tree optimizations
-export const Validators = new ListCompositeType(ValidatorNodeStruct, VALIDATOR_REGISTRY_LIMIT);
+// export const Validators = new ListCompositeType(ValidatorNodeStruct, VALIDATOR_REGISTRY_LIMIT);
+export const Validators = new ListValidatorType(VALIDATOR_REGISTRY_LIMIT);
 export const Balances = new ListUintNum64Type(VALIDATOR_REGISTRY_LIMIT);
 export const RandaoMixes = new VectorCompositeType(Bytes32, EPOCHS_PER_HISTORICAL_VECTOR);
 export const Slashings = new VectorBasicType(Gwei, EPOCHS_PER_SLASHINGS_VECTOR);

packages/ssz/test/lodestarTypes/phase0/validator.ts

@@ -2,7 +2,7 @@ import {ByteViews} from "../../../src/type/abstract";
 import {ContainerNodeStructType} from "../../../src/type/containerNodeStruct";
 import {ValueOfFields} from "../../../src/view/container";
 import * as primitiveSsz from "../primitive/sszTypes";
-import { ValidatorTreeViewDU } from "./viewDU/validatorNodeStruct";
+import { ValidatorTreeViewDU } from "./viewDU/validator";
 import {Node} from "@chainsafe/persistent-merkle-tree";
 
 const {Boolean, Bytes32, UintNum64, BLSPubkey, EpochInf} = primitiveSsz;

packages/ssz/test/lodestarTypes/phase0/viewDU/listValidator.ts

@@ -0,0 +1,191 @@
+import {BranchNode, HashComputation, HashComputationGroup, LeafNode, Node, arrayAtIndex, executeHashComputations, getHashComputations, setNodesAtDepth} from "@chainsafe/persistent-merkle-tree";
+import { ListCompositeType } from "../../../../src/type/listComposite";
+import { ArrayCompositeTreeViewDUCache } from "../../../../src/viewDU/arrayComposite";
+import { ListCompositeTreeViewDU } from "../../../../src/viewDU/listComposite";
+import { ValidatorNodeStructType } from "../validator";
+import { ValidatorTreeViewDU } from "./validator";
+
+/**
+ * Best SIMD implementation is in 512 bits = 64 bytes
+ * If not, hashtree will make a loop inside
+ * Given sha256 operates on a block of 4 bytes, we can hash 16 inputs at once
+ * Each input is 64 bytes
+ * TODO - batch: is 8 better?
+ */
+const PARALLEL_FACTOR = 16;
+
+export class ListValidatorTreeViewDU extends ListCompositeTreeViewDU<ValidatorNodeStructType> {
+  private batchHashComputations: Array<HashComputation[]>;
+  private singleHashComputations: Array<HashComputation[]>;
+  private batchHashRootNodes: Array<Node>;
+  private singleHashRootNode: Node;
+  private batchLevel3Nodes: Array<Node[]>;
+  private singleLevel3Nodes: Node[];
+
+  constructor(
+    readonly type: ListCompositeType<ValidatorNodeStructType>,
+    protected _rootNode: Node,
+    cache?: ArrayCompositeTreeViewDUCache
+  ) {
+    super(type, _rootNode, cache);
+
+    this.batchHashComputations = [];
+    this.singleHashComputations = [];
+    this.batchHashRootNodes = [];
+    this.batchLevel3Nodes = [];
+    this.singleLevel3Nodes = [];
+    for (let i = 0; i < PARALLEL_FACTOR; i++) {
+      // level 3, validator.pubkey
+      const pubkey0 = LeafNode.fromZero();
+      const pubkey1 = LeafNode.fromZero();
+      const pubkey = new BranchNode(pubkey0, pubkey1);
+      let hc: HashComputation = {src0: pubkey0, src1: pubkey1, dest: pubkey};
+      if (i === 0) {
+        arrayAtIndex(this.singleHashComputations, 3).push(hc);
+        this.singleLevel3Nodes.push(pubkey);
+      }
+      arrayAtIndex(this.batchHashComputations, 3).push(hc);
+      arrayAtIndex(this.batchLevel3Nodes, i).push(pubkey);
+
+      // level 2
+      const withdrawalCredential = LeafNode.fromZero();
+      const node20 = new BranchNode(pubkey, withdrawalCredential);
+      hc = {src0: pubkey, src1: withdrawalCredential, dest: node20};
+      if (i === 0) {
+        arrayAtIndex(this.singleHashComputations, 2).push(hc);
+        this.singleLevel3Nodes.push(withdrawalCredential);
+      }
+      arrayAtIndex(this.batchHashComputations, 2).push(hc);
+      arrayAtIndex(this.batchLevel3Nodes, i).push(withdrawalCredential);
+      // effectiveBalance, slashed
+      const effectiveBalance = LeafNode.fromZero();
+      const slashed = LeafNode.fromZero();
+      const node21 = new BranchNode(effectiveBalance, slashed);
+      hc = {src0: effectiveBalance, src1: slashed, dest: node21};
+      if (i === 0) {
+        arrayAtIndex(this.singleHashComputations, 2).push(hc);
+        this.singleLevel3Nodes.push(effectiveBalance);
+        this.singleLevel3Nodes.push(slashed);
+      }
+      arrayAtIndex(this.batchHashComputations, 2).push(hc);
+      arrayAtIndex(this.batchLevel3Nodes, i).push(effectiveBalance);
+      arrayAtIndex(this.batchLevel3Nodes, i).push(slashed);
+      // activationEligibilityEpoch, activationEpoch
+      const activationEligibilityEpoch = LeafNode.fromZero();
+      const activationEpoch = LeafNode.fromZero();
+      const node22 = new BranchNode(activationEligibilityEpoch, activationEpoch);
+      hc = {src0: activationEligibilityEpoch, src1: activationEpoch, dest: node22};
+      if (i === 0) {
+        arrayAtIndex(this.singleHashComputations, 2).push(hc);
+        this.singleLevel3Nodes.push(activationEligibilityEpoch);
+        this.singleLevel3Nodes.push(activationEpoch);
+      }
+      arrayAtIndex(this.batchHashComputations, 2).push(hc);
+      arrayAtIndex(this.batchLevel3Nodes, i).push(activationEligibilityEpoch);
+      arrayAtIndex(this.batchLevel3Nodes, i).push(activationEpoch);
+      // exitEpoch, withdrawableEpoch
+      const exitEpoch = LeafNode.fromZero();
+      const withdrawableEpoch = LeafNode.fromZero();
+      const node23 = new BranchNode(exitEpoch, withdrawableEpoch);
+      hc = {src0: exitEpoch, src1: withdrawableEpoch, dest: node23};
+      if (i === 0) {
+        arrayAtIndex(this.singleHashComputations, 2).push(hc);
+        this.singleLevel3Nodes.push(exitEpoch);
+        this.singleLevel3Nodes.push(withdrawableEpoch);
+      }
+      arrayAtIndex(this.batchHashComputations, 2).push(hc);
+      arrayAtIndex(this.batchLevel3Nodes, i).push(exitEpoch);
+      arrayAtIndex(this.batchLevel3Nodes, i).push(withdrawableEpoch);
+
+      // level 1
+      const node10 = new BranchNode(node20, node21);
+      hc = {src0: node20, src1: node21, dest: node10};
+      if (i === 0) {
+        arrayAtIndex(this.singleHashComputations, 1).push(hc);
+      }
+      arrayAtIndex(this.batchHashComputations, 1).push(hc);
+      const node11 = new BranchNode(node22, node23);
+      hc = {src0: node22, src1: node23, dest: node11};
+      if (i === 0) {
+        arrayAtIndex(this.singleHashComputations, 1).push(hc);
+      }
+      arrayAtIndex(this.batchHashComputations, 1).push(hc);
+
+      // level 0
+      const node00 = new BranchNode(node10, node11);
+      hc = {src0: node10, src1: node11, dest: node00};
+      if (i === 0) {
+        arrayAtIndex(this.singleHashComputations, 0).push(hc);
+        // this.singleHashRootNode = node00;
+      }
+      arrayAtIndex(this.batchHashComputations, 0).push(hc);
+      this.batchHashRootNodes.push(node00);
+    }
+
+    this.singleHashRootNode = this.batchHashRootNodes[0];
+  }
+
+  commit(hashComps: HashComputationGroup | null = null): void {
+    const isOldRootHashed = this._rootNode.h0 !== null;
+    if (this.viewsChanged.size === 0) {
+      if (!isOldRootHashed && hashComps !== null) {
+        getHashComputations(this._rootNode, hashComps.offset, hashComps.byLevel);
+      }
+      return;
+    }
+
+    // TODO - batch: remove this type cast
+    const viewsChanged = Array.from(this.viewsChanged.values()) as ValidatorTreeViewDU[];
+    const endBatch = viewsChanged.length - (viewsChanged.length % PARALLEL_FACTOR);
+    // nodesChanged is sorted by index
+    const nodesChanged: {index: number; node: Node}[] = [];
+    // commit every 16 validators in batch
+    for (let i = 0; i < endBatch; i++) {
+      const indexInBatch = i % PARALLEL_FACTOR;
+      viewsChanged[i].valueToTree(this.batchLevel3Nodes[indexInBatch]);
+      if (indexInBatch === PARALLEL_FACTOR - 1) {
+        executeHashComputations(this.batchHashComputations);
+        // commit all validators in this batch
+        for (let j = PARALLEL_FACTOR - 1; j >= 0; j--) {
+          viewsChanged[i - j].commitToHashObject(this.batchHashRootNodes[PARALLEL_FACTOR - 1 - j]);
+          nodesChanged.push({index: i - j, node: viewsChanged[i - j].node});
+        }
+      }
+    }
+
+    // commit the remaining validators one by one
+    for (let i = endBatch; i < viewsChanged.length; i++) {
+      viewsChanged[i].valueToTree(this.singleLevel3Nodes);
+      executeHashComputations(this.singleHashComputations);
+      viewsChanged[i].commitToHashObject(this.singleHashRootNode);
+      nodesChanged.push({index: i, node: viewsChanged[i].node});
+    }
+
+    // do the remaining commit step the same to parent (ArrayCompositeTreeViewDU)
+    const indexes = nodesChanged.map((entry) => entry.index);
+    const nodes = nodesChanged.map((entry) => entry.node);
+    const chunksNode = this.type.tree_getChunksNode(this._rootNode);
+    const hashCompsThis =
+      hashComps != null && isOldRootHashed
+        ? {
+            byLevel: hashComps.byLevel,
+            offset: hashComps.offset + this.type.tree_chunksNodeOffset(),
+          }
+        : null;
+    const newChunksNode = setNodesAtDepth(chunksNode, this.type.chunkDepth, indexes, nodes, hashCompsThis);
+
+    this._rootNode = this.type.tree_setChunksNode(
+      this._rootNode,
+      newChunksNode,
+      this.dirtyLength ? this._length : null,
+      hashComps
+    );
+
+    if (!isOldRootHashed && hashComps !== null) {
+      getHashComputations(this._rootNode, hashComps.offset, hashComps.byLevel);
+    }
+
+    this.viewsChanged.clear();
+    this.dirtyLength = false;
+  }
+}

packages/ssz/test/unit/lodestarTypes/phase0/listValidator.test.ts

@@ -0,0 +1,38 @@
+import { ListCompositeType } from "../../../../src/type/listComposite";
+import { ValidatorNodeStruct } from "../../../lodestarTypes/phase0/validator";
+import {
+  preset,
+} from "../../../lodestarTypes/params";
+import { ssz } from "../../../lodestarTypes";
+import { expect } from "chai";
+const {VALIDATOR_REGISTRY_LIMIT} = preset;
+
+describe("ListValidator ssz type", function () {
+  const seedValidator = {
+    activationEligibilityEpoch: 10,
+    activationEpoch: 11,
+    exitEpoch: Infinity,
+    slashed: false,
+    withdrawableEpoch: 13,
+    pubkey: Buffer.alloc(48, 100),
+    withdrawalCredentials: Buffer.alloc(32, 100),
+    effectiveBalance: 32000000000,
+  };
+
+  const testCases = [32, 33, 34, 35];
+  const oldValidatorsType = new ListCompositeType(ValidatorNodeStruct, VALIDATOR_REGISTRY_LIMIT);
+  for (const numValidators of testCases) {
+    it (`should commit ${numValidators} validators`, () => {
+      const validators = Array.from({length: numValidators}, (_, i) => ({...seedValidator, withdrawableEpoch: seedValidator.withdrawableEpoch + i}));
+      const oldViewDU = oldValidatorsType.toViewDU(validators);
+      const newViewDU = ssz.phase0.Validators.toViewDU(validators);
+      // modify all validators
+      for (let i = 0; i < numValidators; i++) {
+        oldViewDU.get(i).activationEpoch = 2024;
+        newViewDU.get(i).activationEpoch = 2024;
+      }
+      expect(newViewDU.hashTreeRoot()).to.be.deep.equal(oldViewDU.hashTreeRoot());
+      expect(newViewDU.serialize()).to.be.deep.equal(oldViewDU.serialize());
+    });
+  }
+});

packages/ssz/test/unit/lodestarTypes/phase0/validator.test.ts

@@ -2,7 +2,7 @@ import { BranchNode, LeafNode, Node, subtreeFillToContents } from "@chainsafe/pe
 import {ContainerType} from "../../../../../ssz/src/type/container";
 import {ssz} from "../../../lodestarTypes";
 import {ValidatorType} from "../../../lodestarTypes/phase0/validator";
-import {ValidatorTreeViewDU} from "../../../lodestarTypes/phase0/viewDU/validatorNodeStruct";
+import {ValidatorTreeViewDU} from "../../../lodestarTypes/phase0/viewDU/validator";
 import { expect } from "chai";
 
 const ValidatorContainer = new ContainerType(ValidatorType, {typeName: "Validator", jsonCase: "eth2"});
@@ -16,6 +16,7 @@ describe("Validator ssz types", function () {
     withdrawableEpoch: 13,
     pubkey: Buffer.alloc(48, 100),
     withdrawalCredentials: Buffer.alloc(32, 100),
+    effectiveBalance: 32000000000,
   };
 
   const validators = [

packages/ssz/test/unit/lodestarTypes/phase0/viewDU/validatorNodeStruct.test.ts

@@ -1,5 +1,5 @@
 import { Node, BranchNode, LeafNode, subtreeFillToContents, getNodesAtDepth } from "@chainsafe/persistent-merkle-tree";
-import { validatorToTree } from "../../../../lodestarTypes/phase0/viewDU/validatorNodeStruct";
+import { validatorToTree } from "../../../../lodestarTypes/phase0/viewDU/validator";
 import { HashObject, hashObjectToByteArray } from "@chainsafe/as-sha256";
 import { ValidatorNodeStruct } from "../../../../lodestarTypes/phase0/validator";
 import { expect } from "chai";