Faster fastMsgIdFn using xxhash

package.json

@@ -27,7 +27,7 @@
     "test:spec-fast": "lerna run test:spec-fast --no-bail",
     "test:spec-main": "lerna run test:spec-main --no-bail",
     "benchmark": "yarn benchmark:files 'packages/*/test/perf/**/*.test.ts'",
-    "benchmark:files": "LODESTAR_PRESET=mainnet NODE_OPTIONS='--max-old-space-size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml",
+    "benchmark:files": "LODESTAR_PRESET=mainnet NODE_OPTIONS='--max-old-space-size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml --defaultBranch unstable",
     "release:create-rc": "node scripts/release/create_rc.mjs",
     "release:tag-rc": "node scripts/release/tag_rc.mjs",
     "release:tag-stable": "node scripts/release/tag_stable.mjs",

packages/beacon-node/package.json

@@ -116,7 +116,6 @@
     "@libp2p/mplex": "^5.2.4",
     "@libp2p/peer-id-factory": "^1.0.18",
     "@libp2p/tcp": "^3.1.2",
-    "@multiformats/multiaddr": "^11.0.0",
     "@lodestar/api": "^1.1.0",
     "@lodestar/config": "^1.1.0",
     "@lodestar/db": "^1.1.0",
@@ -127,6 +126,7 @@
     "@lodestar/types": "^1.1.0",
     "@lodestar/utils": "^1.1.0",
     "@lodestar/validator": "^1.1.0",
+    "@multiformats/multiaddr": "^11.0.0",
     "@types/datastore-level": "^3.0.0",
     "buffer-xor": "^2.0.2",
     "cross-fetch": "^3.1.4",
@@ -148,7 +148,8 @@
     "stream-to-it": "^0.2.0",
     "strict-event-emitter-types": "^2.0.0",
     "uint8arraylist": "^2.3.2",
-    "varint": "^6.0.0"
+    "varint": "^6.0.0",
+    "xxhash-wasm": "1.0.1"
   },
   "devDependencies": {
     "@types/bl": "^5.0.1",

packages/beacon-node/src/network/gossip/encoding.ts

@@ -1,4 +1,5 @@
 import {compress, uncompress} from "snappyjs";
+import xxhashFactory from "xxhash-wasm";
 import {Message} from "@libp2p/interface-pubsub";
 import {digest} from "@chainsafe/as-sha256";
 import {intToBytes} from "@lodestar/utils";
@@ -7,13 +8,19 @@ import {RPC} from "@chainsafe/libp2p-gossipsub/message";
 import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
 import {GossipTopicCache} from "./topic.js";
 
+// Load WASM
+const xxhash = await xxhashFactory();
+
+// Use salt to prevent msgId from being mined for collisions
+const h64Seed = BigInt(Math.floor(Math.random() * 1e9));
+
 /**
  * The function used to generate a gossipsub message id
  * We use the first 8 bytes of SHA256(data) for content addressing
  */
 export function fastMsgIdFn(rpcMsg: RPC.IMessage): string {
   if (rpcMsg.data) {
-    return Buffer.from(digest(rpcMsg.data)).subarray(0, 8).toString("hex");
+    return xxhash.h64Raw(rpcMsg.data, h64Seed).toString(16);
   } else {
     return "0000000000000000";
   }

packages/beacon-node/test/perf/network/gossip/fastMsgIdFn.test.ts

@@ -0,0 +1,37 @@
+import {randomBytes} from "node:crypto";
+import xxhashFactory from "xxhash-wasm";
+import {itBench} from "@dapplion/benchmark";
+import {digest} from "@chainsafe/as-sha256";
+
+const hasher = await xxhashFactory();
+
+// Ethereum mainnet case processes 500_000 attestations / epoch (384 sec) = 1302 msg per sec
+
+describe("network / gossip / fastMsgIdFn", () => {
+  const h32Seed = Math.floor(Math.random() * 1e9);
+  const h64Seed = BigInt(Math.floor(Math.random() * 1e9));
+  for (const msgLen of [200, 1000, 10000]) {
+    const msgData = randomBytes(msgLen);
+
+    itBench({
+      id: `fastMsgIdFn sha256 / ${msgLen} bytes`,
+      fn: () => {
+        Buffer.from(digest(msgData)).subarray(0, 8).toString("hex");
+      },
+    });
+
+    itBench({
+      id: `fastMsgIdFn h32 xxhash / ${msgLen} bytes`,
+      fn: () => {
+        hasher.h32Raw(msgData, h32Seed);
+      },
+    });
+
+    itBench({
+      id: `fastMsgIdFn h64 xxhash / ${msgLen} bytes`,
+      fn: () => {
+        hasher.h64Raw(msgData, h64Seed).toString(16);
+      },
+    });
+  }
+});

yarn.lock

@@ -13584,6 +13584,11 @@ xtend@^4.0.0, xtend@^4.0.2, xtend@~4.0.1:
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
   integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
 
+xxhash-wasm@1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/xxhash-wasm/-/xxhash-wasm-1.0.1.tgz#8a0f0eeb3ab76c16bbb889f5acca286b62d98626"
+  integrity sha512-Lc9CTvDrH2vRoiaUzz25q7lRaviMhz90pkx6YxR9EPYtF99yOJnv2cB+CQ0hp/TLoqrUsk8z/W2EN31T568Azw==
+
 y18n@^4.0.0:
   version "4.0.3"
   resolved "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz"