From b420cc8bf436bb9b52d1ddd09843f105c51d8db7 Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Fri, 15 Sep 2023 10:48:03 -0700 Subject: [PATCH 1/8] Start building out basic benchmarking Add bench resource creation script --- bench/.npmrc | 1 + bench/create-resources.js | 49 +++++++++++++++++++++++++++++++++++++++ bench/package.json | 9 +++++++ 3 files changed, 59 insertions(+) create mode 100644 bench/.npmrc create mode 100644 bench/create-resources.js create mode 100644 bench/package.json diff --git a/bench/.npmrc b/bench/.npmrc new file mode 100644 index 00000000..43c97e71 --- /dev/null +++ b/bench/.npmrc @@ -0,0 +1 @@ +package-lock=false diff --git a/bench/create-resources.js b/bench/create-resources.js new file mode 100644 index 00000000..e106f5a8 --- /dev/null +++ b/bench/create-resources.js @@ -0,0 +1,49 @@ +let awsLite = require('../') + +let service = 'dynamodb' +let region = process.env.AWS_REGION || 'us-west-2' +let TableName = process.env.AWS_LITE_TEST_TABLE_NAME || 'aws-lite-test-table' + +async function main () { + let aws = await awsLite({ region }) + + try { + let table = await aws({ + service, + headers: { 'X-Amz-Target': 'DynamoDB_20120810.DescribeTable' }, + payload: { TableName }, + }) + console.log(`Found test table!`, table) + return + } + catch (err) { + if (!err?.__type?.includes('ResourceNotFoundException')) { + throw err + } + console.log('Test table not found, creating table') + } + + let table = await aws({ + service, + headers: { 'X-Amz-Target': 'DynamoDB_20120810.CreateTable' }, + payload: { + TableName, + KeySchema: [ + { + AttributeName: 'id', + KeyType: 'HASH' + } + ], + AttributeDefinitions: [ + { + AttributeName: 'id', + AttributeType: 'S', + } + ], + BillingMode: 'PAY_PER_REQUEST', + }, + }) + console.log(`Created new table: ${TableName} in ${region}`) + console.log(table) +} +main() diff --git a/bench/package.json b/bench/package.json new file mode 100644 index 00000000..396294b4 --- /dev/null +++ b/bench/package.json @@ -0,0 +1,9 @@ +{ + "scripts": { + "create-resources": "node ./create-resources.js" + }, + "dependencies": { + "@aws-sdk/client-dynamodb": "^3.413.0", + "aws-sdk": "^2.1458.0" + } +} From a836c8f909f2b6804ea2d273d1adea4d7cff617a Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Fri, 15 Sep 2023 21:30:14 -0700 Subject: [PATCH 2/8] First cut of speed benchmarking --- bench/_harness.js | 11 ++++ bench/benchmark-speed.js | 105 +++++++++++++++++++++++++++++++++++++++ bench/package.json | 3 ++ 3 files changed, 119 insertions(+) create mode 100644 bench/_harness.js create mode 100755 bench/benchmark-speed.js diff --git a/bench/_harness.js b/bench/_harness.js new file mode 100644 index 00000000..0ec6d62b --- /dev/null +++ b/bench/_harness.js @@ -0,0 +1,11 @@ +/* eslint-disable */ +let report = { times: {}, memory: {} }; +/* $script1 */ +report.memory.start = process.memoryUsage.rss(); +report.times.start = Date.now(); +/* $script2 */ +report.times.end = Date.now(); +report.times.result = report.times.end - report.times.start; +report.memory.end = process.memoryUsage.rss(); +report.memory.result = report.memory.end - report.memory.start; +console.log(JSON.stringify(report, null, 2)); diff --git a/bench/benchmark-speed.js b/bench/benchmark-speed.js new file mode 100755 index 00000000..e324757b --- /dev/null +++ b/bench/benchmark-speed.js @@ -0,0 +1,105 @@ +#! /usr/bin/env node + +let { join } = require('path') +let { execSync } = require('child_process') +let { readFileSync } = require('fs') +let percentile = require('percentile') + +let benchmarkRuns = process.env.AWS_LITE_BENCHMARK_RUNS || 50 +let harness = readFileSync(join(__dirname, '_harness.js')).toString() + +let benchmarks = { + 'aws-lite': { + import: { + script2: `require('../');`, + runs: [], + }, + 'import & instantiate': { + script1: ``, + script2: `let awsLite = require('../'); let client = awsLite({ region: 'us-west-1' });`, + runs: [], + }, + }, + 'AWS SDK v2': { + import: { + script2: ` + process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = true; + require('aws-sdk'); + `, + runs: [], + }, + 'import & instantiate': { + script2: ` + process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = true; + let Aws = require('aws-sdk'); + let aws = new Aws.DynamoDB({ region: 'us-west-1' }); + `, + runs: [], + }, + }, + 'AWS SDK v2 (single client)': { + import: { + script2: ` + process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = true; + require('aws-sdk/clients/dynamodb'); + `, + runs: [], + }, + 'import & instantiate': { + script2: ` + process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = true; + let DynamoDB = require('aws-sdk/clients/dynamodb'); + let aws = new DynamoDB({ region: 'us-west-1' }); + `, + runs: [], + }, + }, + 'AWS SDK v3': { + import: { + script2: `require('@aws-sdk/client-dynamodb');`, + runs: [], + }, + 'import & instantiate': { + script2: ` + let DynamoDB = require('@aws-sdk/client-dynamodb'); + let cmd = new DynamoDB.GetItemCommand({ RequestItem: { 'foo': { Keys: ['bar'] } } }); + `, + runs: [], + }, + } +} + +let benchmarksToRun = [ + // 'import', // Fine, but doesn't really tell us as much import & instantiate + 'import & instantiate', + // TODO: round trip request to DynamoDB +] + +for (let running of benchmarksToRun) { + console.log(`---------- Running ${running} benchmarks ---------- `) + for (let name of Object.keys(benchmarks)) { + console.log(`[${name}]`) + console.log(`Starting ${running} benchmark`) + let { script1 = '', script2 } = benchmarks[name][running] + let start = Date.now() + for (let i = 0; i < benchmarkRuns; i++) { + let run = harness + .replace('/* $script1 */', script1) + .replace('/* $script2 */', script2) + .split('\n').join(' ') + let result = execSync(`node -e "${run}"`, { env: {} }) + benchmarks[name][running].runs.push(JSON.parse(result)) + } + console.log(`Completed ${benchmarkRuns} runs in ${Date.now() - start}ms`) + let times = benchmarks[name][running].runs.map(run => run.times.result) + let avgTime = times.reduce((a, b) => a + b, 0) / times.length + console.log(`- Avg time to complete: ${avgTime}ms`) + console.log(`- p95 time to complete: ${percentile(95, times)}ms`) + + let memory = benchmarks[name][running].runs.map(run => run.memory.result) + let avgMemory = memory.reduce((a, b) => a + b, 0) / times.length / 1024 / 1024 + console.log(`- Avg memory footprint: ${avgMemory}MB`) + console.log(`- p95 memory footprint: ${percentile(95, memory) / 1024 / 1024}MB`) + console.log(``) + } +} diff --git a/bench/package.json b/bench/package.json index 396294b4..7927f1e5 100644 --- a/bench/package.json +++ b/bench/package.json @@ -5,5 +5,8 @@ "dependencies": { "@aws-sdk/client-dynamodb": "^3.413.0", "aws-sdk": "^2.1458.0" + }, + "devDependencies": { + "percentile": "^1.6.0" } } From 35c61fdc9360768794892bd836e6fccc1be7d819 Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Fri, 15 Sep 2023 21:35:24 -0700 Subject: [PATCH 3/8] First cut of size benchmarking Format memory sizes in speed bench --- .gitignore | 1 + bench/_helpers.js | 23 +++++++++++++++++ bench/benchmark-size.js | 34 ++++++++++++++++++++++++++ bench/benchmark-speed.js | 13 +++++----- bench/entry-files/aws-lite.js | 1 + bench/entry-files/aws-sdk-v2-client.js | 1 + bench/entry-files/aws-sdk-v2.js | 1 + bench/entry-files/aws-sdk-v3.js | 1 + bench/package.json | 9 ++++--- 9 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 bench/_helpers.js create mode 100755 bench/benchmark-size.js create mode 100644 bench/entry-files/aws-lite.js create mode 100644 bench/entry-files/aws-sdk-v2-client.js create mode 100644 bench/entry-files/aws-sdk-v2.js create mode 100644 bench/entry-files/aws-sdk-v3.js diff --git a/.gitignore b/.gitignore index dc972671..de99fa8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .nyc_output coverage scratch +tmp diff --git a/bench/_helpers.js b/bench/_helpers.js new file mode 100644 index 00000000..cd96f5a4 --- /dev/null +++ b/bench/_helpers.js @@ -0,0 +1,23 @@ +let KB = 1024 +let MB = KB * KB +function truncate (num) { + let bits = String(num).split('.') + if (bits.length === 1) return bits[0] + return `${bits[0]}.${bits[1].substring(0, 2)}` +} + +function formatSize (num) { + // Measure in KB + if (num < MB) return `${truncate(num / KB)} KB` + // Measure in MB + return `${truncate(num / MB)} MB` +} + +let names = { + 'aws-lite': 'aws-lite', + 'aws-sdk-v2': 'AWS SDK v2', + 'aws-sdk-v2-client': 'AWS SDK v2 (single client)', + 'aws-sdk-v3': 'AWS SDK v3', +} + +module.exports = { formatSize, names } diff --git a/bench/benchmark-size.js b/bench/benchmark-size.js new file mode 100755 index 00000000..2cd7009e --- /dev/null +++ b/bench/benchmark-size.js @@ -0,0 +1,34 @@ +#! /usr/bin/env node +let { mkdirSync, statSync } = require('fs') +let { join } = require('path') +let { build } = require('esbuild') +let { formatSize, names } = require('./_helpers') + +async function main () { + // TODO: run unbundled size on disk benchmarks + console.log(`---------- Running bundle size benchmarks ---------- `) + let entryFileFolder = join(__dirname, 'entry-files') + for (let entry of Object.keys(names)) { + let name = entry.split('.js')[0] + let friendlyName = names[name] + console.log(`[${friendlyName}]`) + + let outDir = join(__dirname, 'tmp', name) + mkdirSync(outDir, { recursive: true }) + let outfile = join(outDir, entry) + + let start = Date.now() + await build({ + entryPoints: [ join(entryFileFolder, entry) ], + bundle: true, + platform: 'node', + format: 'cjs', + outfile, + }) + console.log(`esbuild completed bundling in ${Date.now() - start} ms`) + let { size } = statSync(outfile) + console.log(`Bundle size: ${formatSize(size)}`) + console.log('') + } +} +main() diff --git a/bench/benchmark-speed.js b/bench/benchmark-speed.js index e324757b..1758de16 100755 --- a/bench/benchmark-speed.js +++ b/bench/benchmark-speed.js @@ -4,6 +4,7 @@ let { join } = require('path') let { execSync } = require('child_process') let { readFileSync } = require('fs') let percentile = require('percentile') +let { formatSize } = require('./_helpers') let benchmarkRuns = process.env.AWS_LITE_BENCHMARK_RUNS || 50 let harness = readFileSync(join(__dirname, '_harness.js')).toString() @@ -90,16 +91,16 @@ for (let running of benchmarksToRun) { let result = execSync(`node -e "${run}"`, { env: {} }) benchmarks[name][running].runs.push(JSON.parse(result)) } - console.log(`Completed ${benchmarkRuns} runs in ${Date.now() - start}ms`) + console.log(`Completed ${benchmarkRuns} runs in ${Date.now() - start} ms`) let times = benchmarks[name][running].runs.map(run => run.times.result) let avgTime = times.reduce((a, b) => a + b, 0) / times.length - console.log(`- Avg time to complete: ${avgTime}ms`) - console.log(`- p95 time to complete: ${percentile(95, times)}ms`) + console.log(`- Avg time to complete: ${avgTime} ms`) + console.log(`- p95 time to complete: ${percentile(95, times)} ms`) let memory = benchmarks[name][running].runs.map(run => run.memory.result) - let avgMemory = memory.reduce((a, b) => a + b, 0) / times.length / 1024 / 1024 - console.log(`- Avg memory footprint: ${avgMemory}MB`) - console.log(`- p95 memory footprint: ${percentile(95, memory) / 1024 / 1024}MB`) + let avgMemory = memory.reduce((a, b) => a + b, 0) / times.length + console.log(`- Avg memory footprint: ${formatSize(avgMemory)}`) + console.log(`- p95 memory footprint: ${formatSize(percentile(95, memory))}`) console.log(``) } } diff --git a/bench/entry-files/aws-lite.js b/bench/entry-files/aws-lite.js new file mode 100644 index 00000000..37862936 --- /dev/null +++ b/bench/entry-files/aws-lite.js @@ -0,0 +1 @@ +require('../../') diff --git a/bench/entry-files/aws-sdk-v2-client.js b/bench/entry-files/aws-sdk-v2-client.js new file mode 100644 index 00000000..f6787807 --- /dev/null +++ b/bench/entry-files/aws-sdk-v2-client.js @@ -0,0 +1 @@ +require('aws-sdk/clients/dynamodb') diff --git a/bench/entry-files/aws-sdk-v2.js b/bench/entry-files/aws-sdk-v2.js new file mode 100644 index 00000000..82876f1c --- /dev/null +++ b/bench/entry-files/aws-sdk-v2.js @@ -0,0 +1 @@ +require('aws-sdk') diff --git a/bench/entry-files/aws-sdk-v3.js b/bench/entry-files/aws-sdk-v3.js new file mode 100644 index 00000000..3d14e712 --- /dev/null +++ b/bench/entry-files/aws-sdk-v3.js @@ -0,0 +1 @@ +require('@aws-sdk/client-dynamodb') diff --git a/bench/package.json b/bench/package.json index 7927f1e5..dec371f2 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,12 +1,13 @@ { "scripts": { - "create-resources": "node ./create-resources.js" + "create-resources": "node ./create-resources.js", + "bench": "npm run benchmark", + "benchmark": "./benchmark-speed.js && ./benchmark-size.js" }, "dependencies": { "@aws-sdk/client-dynamodb": "^3.413.0", - "aws-sdk": "^2.1458.0" - }, - "devDependencies": { + "aws-sdk": "^2.1458.0", + "esbuild": "^0.19.0", "percentile": "^1.6.0" } } From 532ca4464f7107f880db11e0268a452dfd467add Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Sun, 17 Sep 2023 09:54:16 -0700 Subject: [PATCH 4/8] Lint --- bench/.eslintrc | 5 +++++ bench/benchmark-size.js | 1 + 2 files changed, 6 insertions(+) create mode 100644 bench/.eslintrc diff --git a/bench/.eslintrc b/bench/.eslintrc new file mode 100644 index 00000000..c96b1668 --- /dev/null +++ b/bench/.eslintrc @@ -0,0 +1,5 @@ +{ + "rules": { + "import/no-unresolved": "off" + } +} diff --git a/bench/benchmark-size.js b/bench/benchmark-size.js index 2cd7009e..7cab54a7 100755 --- a/bench/benchmark-size.js +++ b/bench/benchmark-size.js @@ -2,6 +2,7 @@ let { mkdirSync, statSync } = require('fs') let { join } = require('path') let { build } = require('esbuild') +// eslint-disable-next-line let { formatSize, names } = require('./_helpers') async function main () { From db25c9418f5b4c707ec4f4aa7a66556dc09c9a1f Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Sun, 17 Sep 2023 12:36:06 -0700 Subject: [PATCH 5/8] Add install size benchmark --- bench/_helpers.js | 4 +- bench/benchmark-size.js | 93 ++++++++++++++++++++++++++++++++++------- bench/package.json | 1 + 3 files changed, 82 insertions(+), 16 deletions(-) diff --git a/bench/_helpers.js b/bench/_helpers.js index cd96f5a4..abd637ec 100644 --- a/bench/_helpers.js +++ b/bench/_helpers.js @@ -6,6 +6,8 @@ function truncate (num) { return `${bits[0]}.${bits[1].substring(0, 2)}` } +let roundHalf = num => Math.round(num * 2) / 2 + function formatSize (num) { // Measure in KB if (num < MB) return `${truncate(num / KB)} KB` @@ -20,4 +22,4 @@ let names = { 'aws-sdk-v3': 'AWS SDK v3', } -module.exports = { formatSize, names } +module.exports = { formatSize, names, roundHalf } diff --git a/bench/benchmark-size.js b/bench/benchmark-size.js index 7cab54a7..9f4768c4 100755 --- a/bench/benchmark-size.js +++ b/bench/benchmark-size.js @@ -1,34 +1,97 @@ #! /usr/bin/env node -let { mkdirSync, statSync } = require('fs') +let { mkdirSync, rmSync, statSync, writeFileSync } = require('fs') let { join } = require('path') +let { execSync } = require('child_process') let { build } = require('esbuild') +let folderSize = require('fast-folder-size/sync') + // eslint-disable-next-line -let { formatSize, names } = require('./_helpers') +let { formatSize, names, roundHalf } = require('./_helpers') + +let installCommands = { + 'aws-lite': '@aws-lite/client', // TODO: install '@aws-lite/dynamodb' when ready! + 'aws-sdk-v2': 'aws-sdk', + 'aws-sdk-v2-client': null, + 'aws-sdk-v3': '@aws-sdk/client-dynamodb', +} + +let awsLiteUnbundledSize +let awsLiteBundledSize +let v2installTime +let v2size async function main () { - // TODO: run unbundled size on disk benchmarks console.log(`---------- Running bundle size benchmarks ---------- `) let entryFileFolder = join(__dirname, 'entry-files') - for (let entry of Object.keys(names)) { - let name = entry.split('.js')[0] - let friendlyName = names[name] - console.log(`[${friendlyName}]`) + for (let [ name, friendly ] of Object.entries(names)) { + let isAWSLite = name === 'aws-lite' + let time, unbundledSize, bundledSize + + console.log(`[${friendly}]`) + + /** + * Install + */ + if (name !== 'aws-sdk-v2-client') { + let startInstall = Date.now() + let installDir = join(__dirname, 'tmp', name + '-install') + rmSync(installDir, { recursive: true, force: true }) + mkdirSync(installDir, { recursive: true }) + + // Stub in a package.json + writeFileSync(join(installDir, 'package.json'), '{}') + + console.log(`Installing ${installCommands[name]}...`) + let cmd = `npm i --omit=dev ${installCommands[name]}` + try { + execSync(cmd, { cwd: installDir, stdio: [] }) + } + catch (err) { + console.log(`Installation error`, err) + } + + unbundledSize = folderSize(join(installDir, 'node_modules')) + time = Date.now() - startInstall - let outDir = join(__dirname, 'tmp', name) - mkdirSync(outDir, { recursive: true }) - let outfile = join(outDir, entry) + if (isAWSLite) { + awsLiteUnbundledSize = unbundledSize + } + if (name === 'aws-sdk-v2') { + v2size = unbundledSize + } + console.log(`- Installation time: ${time} ms`) + } + else { + unbundledSize = v2size + console.log(`- Installation time: ${v2installTime} ms`) + } + let larger = '' + if (!isAWSLite) larger = ` (${roundHalf(unbundledSize / awsLiteUnbundledSize)}x larger)` + console.log(`- Unbundled size: ${formatSize(unbundledSize)}${larger}`) - let start = Date.now() + /** + * Bundle + */ + let bundleDir = join(__dirname, 'tmp', name + '-bundle') + mkdirSync(bundleDir, { recursive: true }) + let outfile = join(bundleDir, `${name}-bundle.js`) await build({ - entryPoints: [ join(entryFileFolder, entry) ], + entryPoints: [ join(entryFileFolder, `${name}.js`) ], bundle: true, platform: 'node', format: 'cjs', outfile, }) - console.log(`esbuild completed bundling in ${Date.now() - start} ms`) - let { size } = statSync(outfile) - console.log(`Bundle size: ${formatSize(size)}`) + let stat = statSync(outfile) + bundledSize = stat.size + if (isAWSLite) { + awsLiteBundledSize = bundledSize + larger = '' + } + else { + larger = ` (${roundHalf(bundledSize / awsLiteBundledSize)}x larger)` + } + console.log(`- Bundled size: ${formatSize(bundledSize)}${larger}`) console.log('') } } diff --git a/bench/package.json b/bench/package.json index dec371f2..01fa6073 100644 --- a/bench/package.json +++ b/bench/package.json @@ -8,6 +8,7 @@ "@aws-sdk/client-dynamodb": "^3.413.0", "aws-sdk": "^2.1458.0", "esbuild": "^0.19.0", + "fast-folder-size": "^2.2.0", "percentile": "^1.6.0" } } From 0e8d488ee9d5951f3529cc080cce088a666b0bce Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Sun, 17 Sep 2023 12:48:28 -0700 Subject: [PATCH 6/8] Add `aws-lite` comparisons to speed bench --- bench/benchmark-size.js | 4 ++-- bench/benchmark-speed.js | 37 ++++++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/bench/benchmark-size.js b/bench/benchmark-size.js index 9f4768c4..61df2851 100755 --- a/bench/benchmark-size.js +++ b/bench/benchmark-size.js @@ -66,7 +66,7 @@ async function main () { console.log(`- Installation time: ${v2installTime} ms`) } let larger = '' - if (!isAWSLite) larger = ` (${roundHalf(unbundledSize / awsLiteUnbundledSize)}x larger)` + if (!isAWSLite) larger = ` (~${roundHalf(unbundledSize / awsLiteUnbundledSize)}x larger install size)` console.log(`- Unbundled size: ${formatSize(unbundledSize)}${larger}`) /** @@ -89,7 +89,7 @@ async function main () { larger = '' } else { - larger = ` (${roundHalf(bundledSize / awsLiteBundledSize)}x larger)` + larger = ` (~${roundHalf(bundledSize / awsLiteBundledSize)}x larger bundle size)` } console.log(`- Bundled size: ${formatSize(bundledSize)}${larger}`) console.log('') diff --git a/bench/benchmark-speed.js b/bench/benchmark-speed.js index 1758de16..3536ccf5 100755 --- a/bench/benchmark-speed.js +++ b/bench/benchmark-speed.js @@ -4,7 +4,7 @@ let { join } = require('path') let { execSync } = require('child_process') let { readFileSync } = require('fs') let percentile = require('percentile') -let { formatSize } = require('./_helpers') +let { formatSize, roundHalf } = require('./_helpers') let benchmarkRuns = process.env.AWS_LITE_BENCHMARK_RUNS || 50 let harness = readFileSync(join(__dirname, '_harness.js')).toString() @@ -76,10 +76,19 @@ let benchmarksToRun = [ // TODO: round trip request to DynamoDB ] +let awsLiteAvgSpeed +let awsLiteP95Speed +let awsLiteAvgMemory +let awsLiteP95Memory +let slowerAvg = '', slowerP95 = '', largerAvg = '', largerP95 = '' + for (let running of benchmarksToRun) { console.log(`---------- Running ${running} benchmarks ---------- `) + for (let name of Object.keys(benchmarks)) { + let isAWSLite = name === 'aws-lite' console.log(`[${name}]`) + console.log(`Starting ${running} benchmark`) let { script1 = '', script2 } = benchmarks[name][running] let start = Date.now() @@ -94,13 +103,31 @@ for (let running of benchmarksToRun) { console.log(`Completed ${benchmarkRuns} runs in ${Date.now() - start} ms`) let times = benchmarks[name][running].runs.map(run => run.times.result) let avgTime = times.reduce((a, b) => a + b, 0) / times.length - console.log(`- Avg time to complete: ${avgTime} ms`) - console.log(`- p95 time to complete: ${percentile(95, times)} ms`) + let P95Time = percentile(95, times) + if (!isAWSLite) { + slowerAvg = ` (~${roundHalf(avgTime / awsLiteAvgSpeed)}x slower ${running})` + slowerP95 = ` (~${roundHalf(P95Time / awsLiteP95Speed)}x slower ${running})` + } + else { + awsLiteAvgSpeed = avgTime + awsLiteP95Speed = P95Time + } + console.log(`- Avg time to complete: ${avgTime} ms${slowerAvg}`) + console.log(`- p95 time to complete: ${P95Time} ms${slowerP95}`) let memory = benchmarks[name][running].runs.map(run => run.memory.result) let avgMemory = memory.reduce((a, b) => a + b, 0) / times.length - console.log(`- Avg memory footprint: ${formatSize(avgMemory)}`) - console.log(`- p95 memory footprint: ${formatSize(percentile(95, memory))}`) + let P95Memory = percentile(95, memory) + if (!isAWSLite) { + largerAvg = ` (~${roundHalf(avgMemory / awsLiteAvgMemory)}x more memory)` + largerP95 = ` (~${roundHalf(P95Memory / awsLiteP95Memory)}x more memory)` + } + else { + awsLiteAvgMemory = avgMemory + awsLiteP95Memory = P95Memory + } + console.log(`- Avg memory footprint: ${formatSize(avgMemory)}${largerAvg}`) + console.log(`- p95 memory footprint: ${formatSize(P95Memory)}${largerP95}`) console.log(``) } } From 44c5b68e4f3cc3c2264d506089432550e8a8c8dd Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Sun, 17 Sep 2023 12:55:21 -0700 Subject: [PATCH 7/8] Add benchmark to CI --- .github/workflows/benchmark.yml | 53 +++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..a0bafd43 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,53 @@ +name: Node CI + +# Push tests pushes; PR tests merges +on: [ push ] + +defaults: + run: + shell: bash + working-directory: ./bench + +jobs: + + # Benchmark the build + benchmark: + # Setup + runs-on: ${{ matrix.os }} + strategy: + matrix: + node-version: [ 18.x ] + os: [ ubuntu-latest ] + + # Go + steps: + - name: Check out repo + uses: actions/checkout@v3 + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + + - name: Env + run: | + echo "Event name: ${{ github.event_name }}" + echo "Git ref: ${{ github.ref }}" + echo "GH actor: ${{ github.actor }}" + echo "SHA: ${{ github.sha }}" + VER=`node --version`; echo "Node ver: $VER" + VER=`npm --version`; echo "npm ver: $VER" + echo "OS ver: ${{ runner.os }}" + + - name: Install root deps + run: cd .. && npm install --omit=dev + + - name: Install + run: npm install + + - name: Test + run: npm run bench + # Dummy creds are sufficient until we add live testing + env: + AWS_ACCESS_KEY_ID: 'foo' + AWS_SECRET_ACCESS_KEY: 'bar' From b46379657985ef12011aa82927c210d5df08991d Mon Sep 17 00:00:00 2001 From: Ryan Block Date: Sun, 17 Sep 2023 12:58:40 -0700 Subject: [PATCH 8/8] Ensure aws-lite can run bench with creds (if available) --- bench/benchmark-speed.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/benchmark-speed.js b/bench/benchmark-speed.js index 3536ccf5..a3d97927 100755 --- a/bench/benchmark-speed.js +++ b/bench/benchmark-speed.js @@ -97,7 +97,8 @@ for (let running of benchmarksToRun) { .replace('/* $script1 */', script1) .replace('/* $script2 */', script2) .split('\n').join(' ') - let result = execSync(`node -e "${run}"`, { env: {} }) + let { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = process.env + let result = execSync(`node -e "${run}"`, { env: { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } }) benchmarks[name][running].runs.push(JSON.parse(result)) } console.log(`Completed ${benchmarkRuns} runs in ${Date.now() - start} ms`)