add benchmark.js and github-action-benchmark

.github/workflows/build.yml

@@ -41,3 +41,39 @@ jobs:
       - uses: actions/checkout@v1
       - run: npm install
       - run: npm run test
+
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-node@v1
+      - uses: actions/checkout@v1
+      - run: npm install
+      - run: npm run build
+      - run: npm run benchmarks | tee output.txt
+
+      - name: Set auto-push for benchmarks to true if on master
+        id: auto_push
+        run: |
+          if [$REF == 'refs/heads/master']
+          then
+              echo "::set-output name=auto_push::true"
+          else
+              echo "::set-output name=auto_push::false"
+          fi
+        env:
+          REF: ${{ github.ref }}
+
+      - name: Compare benchmarks
+        uses: rhysd/github-action-benchmark@v1
+        with:
+          tool: 'benchmarkjs'
+          # Where the output from the benchmark tool is stored
+          output-file-path: ./output.txt
+          # Enable alert commit comment
+          comment-on-alert: true
+          # Always leave a commit comment comparing the current benchmark with previous
+          comment-always: true
+          # GitHub API token to make a commit comment
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Push and deploy to GitHub pages branch automatically (if on master)
+          auto-push: ${{ steps.auto_push.outputs.auto_push }}

benchmarks/checkpointing.ts

@@ -1,54 +1,20 @@
 import { pseudoRandomBytes } from 'crypto'
 import { CheckpointTrie } from '../dist'
 
-const iterations = 5000
-const samples = 5
+export const iterTest = async (numOfIter: number) => {
+  const keys: Buffer[] = []
+  const vals: Buffer[] = []
 
-const iterTest = async (numOfIter: number): Promise<Array<number>> => {
-  return new Promise(async (resolve) => {
-    let vals = [] as any
-    let keys = [] as any
-
-    for (let i = 0; i <= numOfIter; i++) {
-      vals.push(pseudoRandomBytes(32))
-      keys.push(pseudoRandomBytes(32))
-    }
-
-    let hrstart = process.hrtime()
-    let numOfOps = 0
-    let trie = new CheckpointTrie()
-    for (let i = 0; i < numOfIter; i++) {
-      trie.checkpoint()
-      await trie.put(vals[i], keys[i])
-      await trie.get(Buffer.from('test'))
-      numOfOps++
-      if (numOfOps === numOfIter) {
-        const hrend = process.hrtime(hrstart)
-        resolve(hrend)
-      }
-      trie.commit()
-    }
-  })
-}
+  for (let i = 0; i <= numOfIter; i++) {
+    keys.push(pseudoRandomBytes(32))
+    vals.push(pseudoRandomBytes(32))
+  }
 
-const go = async () => {
-  let i = 1
-  let avg = [0, 0]
+  const trie = new CheckpointTrie()
 
-  console.log(`Benchmark 'checkpointing' starting...`)
-  while (i <= samples) {
-    console.log(`Sample ${i} with ${iterations} iterations.`)
-    const hrend = await iterTest(iterations)
-    avg[0] += hrend[0]
-    avg[1] += hrend[1]
-    // console.log('benchmarks/checkpointing.ts | execution time: %ds %dms', hrend[0], (hrend[1] / 1000000).toFixed(3))
-    i++
+  for (let i = 0; i < numOfIter; i++) {
+    trie.checkpoint()
+    await trie.put(keys[i], vals[i])
+    await trie.commit()
   }
-  console.log(
-    'benchmarks/checkpointing.ts | average execution time: %ds %dms',
-    avg[0] / samples,
-    (avg[1] / 1000000 / samples).toFixed(3),
-  )
 }
-
-go()

benchmarks/index.ts

@@ -1,2 +1,48 @@
-require('./checkpointing')
-require('./random')
+import Benchmark = require('benchmark')
+import { runTrie } from './random'
+import { iterTest } from './checkpointing'
+
+const suite = new Benchmark.Suite()
+
+// random.ts
+// Test ID is defined as: `pair_count`-`era_size`-`key_size`-`value_type`
+// where value_type = symmetric ? 'mir' : 'ran'
+// The standard secure-trie test is `1k-9-32-ran`
+// https://eth.wiki/en/fundamentals/benchmarks#results-1
+suite
+  .add('1k-3-32-ran', async () => {
+    await runTrie(3, false)
+  })
+  .add('1k-5-32-ran', async () => {
+    await runTrie(5, false)
+  })
+  .add('1k-9-32-ran', async () => {
+    await runTrie(9, false)
+  })
+  .add('1k-1k-32-ran', async () => {
+    await runTrie(1000, false)
+  })
+  .add('1k-1k-32-mir', async () => {
+    await runTrie(1000, true)
+  })
+
+// checkpointing.ts
+suite
+  .add('Checkpointing: 100 iterations', async () => {
+    await iterTest(100)
+  })
+  .add('Checkpointing: 500 iterations', async () => {
+    await iterTest(500)
+  })
+  .add('Checkpointing: 1000 iterations', async () => {
+    await iterTest(1000)
+  })
+  .add('Checkpointing: 5000 iterations', async () => {
+    await iterTest(5000)
+  })
+
+suite
+  .on('cycle', (event: any) => {
+    console.log(String(event.target))
+  })
+  .run()

benchmarks/random.ts

@@ -1,46 +1,30 @@
-// https://github.com/ethereum/wiki/wiki/Benchmarks
-const crypto = require('crypto')
-import { CheckpointTrie } from '../dist'
+'use strict'
+import { keccak256 } from 'ethereumjs-util'
+import { CheckpointTrie as Trie } from '../dist'
 
-const ROUNDS = 50000
-const SYMMETRIC = true
-const ERA_SIZE = 10000
+// References:
+// https://eth.wiki/en/fundamentals/benchmarks#the-trie
+// https://gist.github.com/heikoheiko/0fa2b322560ba7794f22
 
-const trie = new CheckpointTrie()
+const ROUNDS = 1000
+const KEY_SIZE = 32
 
-const run = async (): Promise<void> => {
-  console.log(`Benchmark 'random' starting...`)
-  let i = 0
-  while (i <= ROUNDS) {
-    let key = crypto.randomBytes(32)
+export const runTrie = async (eraSize = 9, symmetric = false) => {
+  const trie = new Trie()
+  let key = Buffer.alloc(KEY_SIZE)
 
-    const genRoot = () => {
-      if (i % ERA_SIZE === 0) {
-        key = trie.root
-        console.log(`${i} rounds.`)
-      }
-    }
+  for (let i = 0; i <= ROUNDS; i++) {
+    key = keccak256(key)
 
-    if (SYMMETRIC) {
+    if (symmetric) {
       await trie.put(key, key)
-      genRoot()
     } else {
-      const value = crypto.randomBytes(32)
-      await trie.put(key, value)
-      genRoot()
+      const val = keccak256(key)
+      await trie.put(key, val)
     }
 
-    i++
+    if (i % eraSize === 0) {
+      key = trie.root
+    }
   }
 }
-
-const go = async () => {
-  const testName = `benchmarks/random.ts | rounds: ${ROUNDS}, ERA_SIZE: ${ERA_SIZE}, ${
-    SYMMETRIC ? 'sys' : 'rand'
-  }`
-  console.time(testName)
-  await run()
-  console.timeEnd(testName)
-}
-
-go()

package.json

@@ -10,7 +10,7 @@
   ],
   "browser": "dist.browser/index.js",
   "scripts": {
-    "benchmarks": "npm run build && ts-node benchmarks/checkpointing.ts && ts-node benchmarks/random.ts",
+    "benchmarks": "npm run build && node -r ts-node/register --max-old-space-size=8024 benchmarks",
     "profiling": "npm run build && tsc --target ES5 benchmarks/random.ts && 0x benchmarks/random.js",
     "build": "tsc -p tsconfig.prod.json && tsc -p tsconfig.browser.json",
     "prepublishOnly": "npm run test && npm run build",
@@ -64,8 +64,10 @@
     "@ethereumjs/config-prettier": "^1.1.1",
     "@ethereumjs/config-tsc": "^1.1.1",
     "@ethereumjs/config-tslint": "^1.1.1",
+    "@types/benchmark": "^1.0.33",
     "@types/bn.js": "^4.11.6",
     "@types/tape": "^4.2.34",
+    "benchmark": "^2.1.4",
     "husky": "^4.2.3",
     "karma": "^4.4.1",
     "karma-chrome-launcher": "^3.1.0",