feat: batch hash using as-sha256

packages/as-sha256/src/index.ts

@@ -235,6 +235,35 @@ export function batchHash4HashObjectInputs(inputs: HashObject[]): HashObject[] {
   return [output0, output1, output2, output3];
 }
 
+/**
+ * Hash an input into preallocated input using batch if possible.
+ */
+export function hashInto(input: Uint8Array, output: Uint8Array): void {
+  if (input.length % 64 !== 0) {
+    throw new Error(`Invalid input length ${input.length}`);
+  }
+  if (input.length !== output.length * 2) {
+    throw new Error(`Invalid output length ${output.length}`);
+  }
+  // for every 64 x 4 = 256 bytes, do the batch hash
+  const endBatch = Math.floor(input.length / 256);
+  for (let i = 0; i < endBatch; i++) {
+    inputUint8Array.set(input.subarray(i * 256, (i + 1) * 256), 0);
+    ctx.batchHash4UintArray64s(wasmOutputValue);
+    output.set(outputUint8Array.subarray(0, 128), i * 128);
+  }
+
+  const numHashed = endBatch * 4;
+  const remainingHash = Math.floor((input.length % 256) / 64);
+  const inputOffset = numHashed * 64;
+  const outputOffset = numHashed * 32;
+  for (let i = 0; i < remainingHash; i++) {
+    inputUint8Array.set(input.subarray(inputOffset + i * 64, inputOffset + (i + 1) * 64), 0);
+    ctx.digest64(wasmInputValue, wasmOutputValue);
+    output.set(outputUint8Array.subarray(0, 32), outputOffset + i * 32);
+  }
+}
+
 function update(data: Uint8Array): void {
   const INPUT_LENGTH = ctx.INPUT_LENGTH;
   if (data.length > INPUT_LENGTH) {

packages/as-sha256/test/unit/simd.test.ts

@@ -44,4 +44,23 @@ describe("Test SIMD implementation of as-sha256", () => {
       expect(output).to.be.deep.equal(expectedOutput, "incorrect batchHash4UintArray64s result " + i);
     }
   });
+
+  const numHashes = [4, 5, 6, 7];
+  for (const numHash of numHashes) {
+    it(`hashInto ${numHash} hashes`, () => {
+      const inputs = Array.from({length: numHash}, () => crypto.randomBytes(64));
+      const input = new Uint8Array(numHash * 64);
+      for (let i = 0; i < numHash; i++) {
+        input.set(inputs[i], i * 64);
+      }
+      const output = new Uint8Array(numHash * 32);
+
+      sha256.hashInto(input, output);
+
+      const expectedOutputs = Array.from({length: numHash}, (_, i) => sha256.digest64(inputs[i]));
+      for (let i = 0; i < numHash; i++) {
+        expect(output.subarray(i * 32, (i + 1) * 32)).to.be.deep.equal(expectedOutputs[i]);
+      }
+    });
+  }
 });

packages/persistent-merkle-tree/package.json

@@ -46,12 +46,6 @@
   "homepage": "https://github.com/ChainSafe/persistent-merkle-tree#readme",
   "dependencies": {
     "@chainsafe/as-sha256": "0.4.2",
-    "@chainsafe/hashtree": "1.0.0",
     "@noble/hashes": "^1.3.0"
-  },
-  "peerDependencies": {
-    "@chainsafe/hashtree-linux-x64-gnu": "1.0.0",
-    "@chainsafe/hashtree-linux-arm64-gnu": "1.0.0",
-    "@chainsafe/hashtree-darwin-arm64": "1.0.0"
   }
 }

packages/persistent-merkle-tree/src/hasher/as-sha256.ts

@@ -1,13 +1,45 @@
-import {digest2Bytes32, digest64HashObjects, HashObject, batchHash4HashObjectInputs} from "@chainsafe/as-sha256";
+import {digest2Bytes32, digest64HashObjects, HashObject, batchHash4HashObjectInputs, hashInto} from "@chainsafe/as-sha256";
 import type {Hasher} from "./types";
 import {HashComputation, Node} from "../node";
 
+// each validator needs to digest 8 chunks of 32 bytes = 4 hashes
+// support up to 4 validators
+const MAX_HASH = 16;
+const MAX_INPUT_SIZE = MAX_HASH * 64;
+const buffer = new Uint8Array(MAX_INPUT_SIZE);
+
 export const hasher: Hasher = {
   name: "as-sha256",
   digest64: digest2Bytes32,
   digest64HashObjects,
+  // given nLevel = 3
+  // digest multiple of 8 chunks = 256 bytes
+  // the result is multiple of 1 chunk = 32 bytes
+  // this is the same to hashTreeRoot() of multiple validators
   digestNLevelUnsafe(data: Uint8Array, nLevel: number): Uint8Array {
-    throw new Error("Not implemented");
+    let inputLength = data.length;
+    const bytesInBatch = Math.pow(2, nLevel) * 32;
+    if (nLevel < 1) {
+      throw new Error(`Invalid nLevel, expect to be greater than 0, got ${nLevel}`);
+    }
+    if (inputLength % bytesInBatch !== 0) {
+      throw new Error(`Invalid input length, expect to be multiple of ${bytesInBatch} for nLevel ${nLevel}, got ${inputLength}`);
+    }
+    if (inputLength > MAX_INPUT_SIZE) {
+      throw new Error(`Invalid input length, expect to be less than ${MAX_INPUT_SIZE}, got ${inputLength}`);
+    }
+
+    buffer.set(data, 0);
+    for (let i = nLevel; i > 0; i--) {
+      const outputLength = Math.floor(inputLength / 2);
+      const hashInput = buffer.subarray(0, inputLength);
+      const hashOutput = buffer.subarray(0, outputLength);
+      hashInto(hashInput, hashOutput);
+      inputLength = outputLength
+    }
+
+    // the result is unsafe as it will be modified later, consumer should save the result if needed
+    return buffer.subarray(0, inputLength);
   },
   batchHashObjects: (inputs: HashObject[]) => {
     // as-sha256 uses SIMD for batch hash

packages/persistent-merkle-tree/src/hasher/index.ts

@@ -1,7 +1,6 @@
 import {Hasher} from "./types";
 // import {hasher as nobleHasher} from "./noble";
-// import {hasher as csHasher} from "./as-sha256";
-import {hasher as hashtreeHasher} from "./hashtree";
+import {hasher as csHasher} from "./as-sha256";
 
 export * from "./types";
 export * from "./util";
@@ -13,10 +12,7 @@ export * from "./util";
  */
 // export let hasher: Hasher = nobleHasher;
 // For testing purposes, we use the as-sha256 hasher
-// export let hasher: Hasher = csHasher;
-
-// For testing purposes, we use the hashtree hasher
-export let hasher: Hasher = hashtreeHasher;
+export let hasher: Hasher = csHasher;
 
 /**
  * Set the hasher to be used across the SSZ codebase

packages/persistent-merkle-tree/test/unit/hasher.test.ts

@@ -2,11 +2,11 @@ import {expectEqualHex} from "../utils/expectHex";
 import {uint8ArrayToHashObject, hashObjectToUint8Array} from "../../src/hasher/util";
 import {hasher as nobleHasher} from "../../src/hasher/noble";
 import {hasher as asSha256Hasher} from "../../src/hasher/as-sha256";
-import {hasher as hashtreeHasher} from "../../src/hasher/hashtree";
 import {linspace} from "../utils/misc";
 import {buildComparisonTrees} from "../utils/tree";
+import { LeafNode, subtreeFillToContents } from "../../src";
 
-const hashers = [hashtreeHasher, asSha256Hasher, nobleHasher];
+const hashers = [asSha256Hasher, nobleHasher];
 
 describe("hashers", function () {
   describe("digest64 vs digest64HashObjects methods should be the same", () => {
@@ -30,9 +30,7 @@ describe("hashers", function () {
     const root2 = Buffer.alloc(32, 0xff);
     const hash1 = nobleHasher.digest64(root1, root2);
     const hash2 = asSha256Hasher.digest64(root1, root2);
-    const hash3 = hashtreeHasher.digest64(root1, root2);
     expectEqualHex(hash1, hash2);
-    expectEqualHex(hash1, hash3);
   });
 
   it("all hashers should return the same values from digest64HashObjects", () => {
@@ -42,9 +40,7 @@ describe("hashers", function () {
     const hashObject2 = uint8ArrayToHashObject(root2);
     const hash1 = hashObjectToUint8Array(nobleHasher.digest64HashObjects(hashObject1, hashObject2));
     const hash2 = hashObjectToUint8Array(asSha256Hasher.digest64HashObjects(hashObject1, hashObject2));
-    const hash3 = hashObjectToUint8Array(hashtreeHasher.digest64HashObjects(hashObject1, hashObject2));
     expectEqualHex(hash1, hash2);
-    expectEqualHex(hash1, hash3);
   });
 
   it("all hashers should return the same values from batchHashObjects", () => {
@@ -53,10 +49,8 @@ describe("hashers", function () {
       .map(uint8ArrayToHashObject);
     const results1 = nobleHasher.batchHashObjects(hashObjects).map(hashObjectToUint8Array);
     const results2 = asSha256Hasher.batchHashObjects(hashObjects).map(hashObjectToUint8Array);
-    const results3 = hashtreeHasher.batchHashObjects(hashObjects).map(hashObjectToUint8Array);
     Object.values(results1).forEach((result1, i) => {
       expectEqualHex(result1, results2[i]);
-      expectEqualHex(result1, results3[i]);
     });
   });
 
@@ -72,4 +66,19 @@ describe("hashers", function () {
   });
 });
 
+describe("as-sha256 hasher", function () {
+  const numValidators = [1, 2, 3, 4];
+  for (const numValidator of numValidators) {
+    it (`digestNLevelUnsafe ${numValidator} validators = ${8 * numValidator} chunk(s)`, () => {
+      const nodes = Array.from({length: 8 * numValidator}, (_, i) => LeafNode.fromRoot(Buffer.alloc(32, i + numValidator)));
+      const hashInput = Buffer.concat(nodes.map((node) => node.root));
+      const hashOutput = asSha256Hasher.digestNLevelUnsafe(hashInput, 3);
+      for (let i = 0; i < numValidator; i++) {
+        const root = subtreeFillToContents(nodes.slice(i * 8, (i + 1) * 8), 3).root;
+        expectEqualHex(hashOutput.subarray(i * 32, (i + 1) * 32), root);
+      }
+    });
+  }
+});
+
 // TODO - batch: test more methods

setHasher.mjs

@@ -1,5 +1,5 @@
 // Set the hasher to hashtree
 // Used to run benchmarks with with visibility into hashtree performance, useful for Lodestar
 import {setHasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/index.js";
-import {hasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/hashtree.js";
+import {hasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/as-sha256.js";
 setHasher(hasher);