From e50db5e932078069297223a0bb945749075e82a7 Mon Sep 17 00:00:00 2001 From: Ryan Ghods Date: Thu, 20 Aug 2020 19:28:45 -0700 Subject: [PATCH] add benchmark.js and github-action-benchmark --- .github/workflows/build.yml | 36 +++++++++++++++++++++++ benchmarks/checkpointing.ts | 58 ++++++++----------------------------- benchmarks/index.ts | 50 ++++++++++++++++++++++++++++++-- benchmarks/random.ts | 54 ++++++++++++---------------------- package.json | 4 ++- 5 files changed, 118 insertions(+), 84 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4633006..bfd969b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,3 +41,39 @@ jobs: - uses: actions/checkout@v1 - run: npm install - run: npm run test + + benchmarks: + runs-on: ubuntu-latest + steps: + - uses: actions/setup-node@v1 + - uses: actions/checkout@v1 + - run: npm install + - run: npm run build + - run: npm run benchmarks | tee output.txt + + - name: Set auto-push for benchmarks to true if on master + id: auto_push + run: | + if [$REF == 'refs/heads/master'] + then + echo "::set-output name=auto_push::true" + else + echo "::set-output name=auto_push::false" + fi + env: + REF: ${{ github.ref }} + + - name: Compare benchmarks + uses: rhysd/github-action-benchmark@v1 + with: + tool: 'benchmarkjs' + # Where the output from the benchmark tool is stored + output-file-path: ./output.txt + # Enable alert commit comment + comment-on-alert: true + # Always leave a commit comment comparing the current benchmark with previous + comment-always: true + # GitHub API token to make a commit comment + github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy to GitHub pages branch automatically (if on master) + auto-push: ${{ steps.auto_push.outputs.auto_push }} diff --git a/benchmarks/checkpointing.ts b/benchmarks/checkpointing.ts index ffad930..fdb0060 100644 --- a/benchmarks/checkpointing.ts +++ b/benchmarks/checkpointing.ts @@ -1,54 +1,20 @@ import { pseudoRandomBytes } from 'crypto' import { CheckpointTrie } from '../dist' -const iterations = 5000 -const samples = 5 +export const iterTest = async (numOfIter: number) => { + const keys: Buffer[] = [] + const vals: Buffer[] = [] -const iterTest = async (numOfIter: number): Promise> => { - return new Promise(async (resolve) => { - let vals = [] as any - let keys = [] as any - - for (let i = 0; i <= numOfIter; i++) { - vals.push(pseudoRandomBytes(32)) - keys.push(pseudoRandomBytes(32)) - } - - let hrstart = process.hrtime() - let numOfOps = 0 - let trie = new CheckpointTrie() - for (let i = 0; i < numOfIter; i++) { - trie.checkpoint() - await trie.put(vals[i], keys[i]) - await trie.get(Buffer.from('test')) - numOfOps++ - if (numOfOps === numOfIter) { - const hrend = process.hrtime(hrstart) - resolve(hrend) - } - trie.commit() - } - }) -} + for (let i = 0; i <= numOfIter; i++) { + keys.push(pseudoRandomBytes(32)) + vals.push(pseudoRandomBytes(32)) + } -const go = async () => { - let i = 1 - let avg = [0, 0] + const trie = new CheckpointTrie() - console.log(`Benchmark 'checkpointing' starting...`) - while (i <= samples) { - console.log(`Sample ${i} with ${iterations} iterations.`) - const hrend = await iterTest(iterations) - avg[0] += hrend[0] - avg[1] += hrend[1] - // console.log('benchmarks/checkpointing.ts | execution time: %ds %dms', hrend[0], (hrend[1] / 1000000).toFixed(3)) - i++ + for (let i = 0; i < numOfIter; i++) { + trie.checkpoint() + await trie.put(keys[i], vals[i]) + await trie.commit() } - console.log( - 'benchmarks/checkpointing.ts | average execution time: %ds %dms', - avg[0] / samples, - (avg[1] / 1000000 / samples).toFixed(3), - ) } - -go() diff --git a/benchmarks/index.ts b/benchmarks/index.ts index e9424c7..94991c0 100644 --- a/benchmarks/index.ts +++ b/benchmarks/index.ts @@ -1,2 +1,48 @@ -require('./checkpointing') -require('./random') +import Benchmark = require('benchmark') +import { runTrie } from './random' +import { iterTest } from './checkpointing' + +const suite = new Benchmark.Suite() + +// random.ts +// Test ID is defined as: `pair_count`-`era_size`-`key_size`-`value_type` +// where value_type = symmetric ? 'mir' : 'ran' +// The standard secure-trie test is `1k-9-32-ran` +// https://eth.wiki/en/fundamentals/benchmarks#results-1 +suite + .add('1k-3-32-ran', async () => { + await runTrie(3, false) + }) + .add('1k-5-32-ran', async () => { + await runTrie(5, false) + }) + .add('1k-9-32-ran', async () => { + await runTrie(9, false) + }) + .add('1k-1k-32-ran', async () => { + await runTrie(1000, false) + }) + .add('1k-1k-32-mir', async () => { + await runTrie(1000, true) + }) + +// checkpointing.ts +suite + .add('Checkpointing: 100 iterations', async () => { + await iterTest(100) + }) + .add('Checkpointing: 500 iterations', async () => { + await iterTest(500) + }) + .add('Checkpointing: 1000 iterations', async () => { + await iterTest(1000) + }) + .add('Checkpointing: 5000 iterations', async () => { + await iterTest(5000) + }) + +suite + .on('cycle', (event: any) => { + console.log(String(event.target)) + }) + .run() diff --git a/benchmarks/random.ts b/benchmarks/random.ts index 40dcf28..943aa6f 100644 --- a/benchmarks/random.ts +++ b/benchmarks/random.ts @@ -1,46 +1,30 @@ -// https://github.com/ethereum/wiki/wiki/Benchmarks -const crypto = require('crypto') -import { CheckpointTrie } from '../dist' +'use strict' +import { keccak256 } from 'ethereumjs-util' +import { CheckpointTrie as Trie } from '../dist' -const ROUNDS = 50000 -const SYMMETRIC = true -const ERA_SIZE = 10000 +// References: +// https://eth.wiki/en/fundamentals/benchmarks#the-trie +// https://gist.github.com/heikoheiko/0fa2b322560ba7794f22 -const trie = new CheckpointTrie() +const ROUNDS = 1000 +const KEY_SIZE = 32 -const run = async (): Promise => { - console.log(`Benchmark 'random' starting...`) - let i = 0 - while (i <= ROUNDS) { - let key = crypto.randomBytes(32) +export const runTrie = async (eraSize = 9, symmetric = false) => { + const trie = new Trie() + let key = Buffer.alloc(KEY_SIZE) - const genRoot = () => { - if (i % ERA_SIZE === 0) { - key = trie.root - console.log(`${i} rounds.`) - } - } + for (let i = 0; i <= ROUNDS; i++) { + key = keccak256(key) - if (SYMMETRIC) { + if (symmetric) { await trie.put(key, key) - genRoot() } else { - const value = crypto.randomBytes(32) - await trie.put(key, value) - genRoot() + const val = keccak256(key) + await trie.put(key, val) } - i++ + if (i % eraSize === 0) { + key = trie.root + } } } - -const go = async () => { - const testName = `benchmarks/random.ts | rounds: ${ROUNDS}, ERA_SIZE: ${ERA_SIZE}, ${ - SYMMETRIC ? 'sys' : 'rand' - }` - console.time(testName) - await run() - console.timeEnd(testName) -} - -go() diff --git a/package.json b/package.json index feecfa3..079b47a 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ ], "browser": "dist.browser/index.js", "scripts": { - "benchmarks": "npm run build && ts-node benchmarks/checkpointing.ts && ts-node benchmarks/random.ts", + "benchmarks": "npm run build && node -r ts-node/register --max-old-space-size=8024 benchmarks", "profiling": "npm run build && tsc --target ES5 benchmarks/random.ts && 0x benchmarks/random.js", "build": "tsc -p tsconfig.prod.json && tsc -p tsconfig.browser.json", "prepublishOnly": "npm run test && npm run build", @@ -64,8 +64,10 @@ "@ethereumjs/config-prettier": "^1.1.1", "@ethereumjs/config-tsc": "^1.1.1", "@ethereumjs/config-tslint": "^1.1.1", + "@types/benchmark": "^1.0.33", "@types/bn.js": "^4.11.6", "@types/tape": "^4.2.34", + "benchmark": "^2.1.4", "husky": "^4.2.3", "karma": "^4.4.1", "karma-chrome-launcher": "^3.1.0",