Bench

.github/workflows/benchmark.yml

@@ -0,0 +1,53 @@
+name: Node CI
+
+# Push tests pushes; PR tests merges
+on: [ push ]
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./bench
+
+jobs:
+
+  # Benchmark the build
+  benchmark:
+    # Setup
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        node-version: [ 18.x ]
+        os: [ ubuntu-latest ]
+
+    # Go
+    steps:
+    - name: Check out repo
+      uses: actions/checkout@v3
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Env
+      run: |
+        echo "Event name: ${{ github.event_name }}"
+        echo "Git ref:    ${{ github.ref }}"
+        echo "GH actor:   ${{ github.actor }}"
+        echo "SHA:        ${{ github.sha }}"
+        VER=`node --version`; echo "Node ver:   $VER"
+        VER=`npm --version`; echo "npm ver:    $VER"
+        echo "OS ver:     ${{ runner.os }}"
+
+    - name: Install root deps
+      run: cd .. && npm install --omit=dev
+
+    - name: Install
+      run: npm install
+
+    - name: Test
+      run: npm run bench
+      # Dummy creds are sufficient until we add live testing
+      env:
+        AWS_ACCESS_KEY_ID: 'foo'
+        AWS_SECRET_ACCESS_KEY: 'bar'

.gitignore

@@ -1,3 +1,4 @@
 .nyc_output
 coverage
 scratch
+tmp

bench/.eslintrc

@@ -0,0 +1,5 @@
+{
+  "rules": {
+    "import/no-unresolved": "off"
+  }
+}

bench/.npmrc

@@ -0,0 +1 @@
+package-lock=false

bench/_harness.js

@@ -0,0 +1,11 @@
+/* eslint-disable */
+let report = { times: {}, memory: {} };
+/* $script1 */
+report.memory.start = process.memoryUsage.rss();
+report.times.start = Date.now();
+/* $script2 */
+report.times.end = Date.now();
+report.times.result = report.times.end - report.times.start;
+report.memory.end = process.memoryUsage.rss();
+report.memory.result = report.memory.end - report.memory.start;
+console.log(JSON.stringify(report, null, 2));

bench/_helpers.js

@@ -0,0 +1,25 @@
+let KB = 1024
+let MB = KB * KB
+function truncate (num) {
+  let bits = String(num).split('.')
+  if (bits.length === 1) return bits[0]
+  return `${bits[0]}.${bits[1].substring(0, 2)}`
+}
+
+let roundHalf = num =>  Math.round(num * 2) / 2
+
+function formatSize (num) {
+  // Measure in KB
+  if (num < MB) return `${truncate(num / KB)} KB`
+  // Measure in MB
+  return `${truncate(num / MB)} MB`
+}
+
+let names = {
+  'aws-lite': 'aws-lite',
+  'aws-sdk-v2': 'AWS SDK v2',
+  'aws-sdk-v2-client': 'AWS SDK v2 (single client)',
+  'aws-sdk-v3': 'AWS SDK v3',
+}
+
+module.exports = { formatSize, names, roundHalf }

bench/benchmark-size.js

@@ -0,0 +1,98 @@
+#! /usr/bin/env node
+let { mkdirSync, rmSync, statSync, writeFileSync } = require('fs')
+let { join } = require('path')
+let { execSync }  = require('child_process')
+let { build } = require('esbuild')
+let folderSize = require('fast-folder-size/sync')
+
+// eslint-disable-next-line
+let { formatSize, names, roundHalf } = require('./_helpers')
+
+let installCommands = {
+  'aws-lite': '@aws-lite/client', // TODO: install '@aws-lite/dynamodb' when ready!
+  'aws-sdk-v2': 'aws-sdk',
+  'aws-sdk-v2-client': null,
+  'aws-sdk-v3': '@aws-sdk/client-dynamodb',
+}
+
+let awsLiteUnbundledSize
+let awsLiteBundledSize
+let v2installTime
+let v2size
+
+async function main () {
+  console.log(`---------- Running bundle size benchmarks ---------- `)
+  let entryFileFolder = join(__dirname, 'entry-files')
+  for (let [ name, friendly ] of Object.entries(names)) {
+    let isAWSLite = name === 'aws-lite'
+    let time, unbundledSize, bundledSize
+
+    console.log(`[${friendly}]`)
+
+    /**
+     * Install
+     */
+    if (name !== 'aws-sdk-v2-client') {
+      let startInstall = Date.now()
+      let installDir = join(__dirname, 'tmp', name + '-install')
+      rmSync(installDir, { recursive: true, force: true })
+      mkdirSync(installDir, { recursive: true })
+
+      // Stub in a package.json
+      writeFileSync(join(installDir, 'package.json'), '{}')
+
+      console.log(`Installing ${installCommands[name]}...`)
+      let cmd = `npm i --omit=dev ${installCommands[name]}`
+      try {
+        execSync(cmd, { cwd: installDir, stdio: [] })
+      }
+      catch (err) {
+        console.log(`Installation error`, err)
+      }
+
+      unbundledSize = folderSize(join(installDir, 'node_modules'))
+      time = Date.now() - startInstall
+
+      if (isAWSLite) {
+        awsLiteUnbundledSize = unbundledSize
+      }
+      if (name === 'aws-sdk-v2') {
+        v2size = unbundledSize
+      }
+      console.log(`- Installation time: ${time} ms`)
+    }
+    else {
+      unbundledSize = v2size
+      console.log(`- Installation time: ${v2installTime} ms`)
+    }
+    let larger = ''
+    if (!isAWSLite) larger = ` (~${roundHalf(unbundledSize / awsLiteUnbundledSize)}x larger install size)`
+    console.log(`- Unbundled size: ${formatSize(unbundledSize)}${larger}`)
+
+    /**
+     * Bundle
+     */
+    let bundleDir = join(__dirname, 'tmp', name + '-bundle')
+    mkdirSync(bundleDir, { recursive: true })
+    let outfile = join(bundleDir, `${name}-bundle.js`)
+    await build({
+      entryPoints: [ join(entryFileFolder, `${name}.js`) ],
+      bundle: true,
+      platform: 'node',
+      format: 'cjs',
+      outfile,
+    })
+    let stat = statSync(outfile)
+    bundledSize = stat.size
+    if (isAWSLite) {
+      awsLiteBundledSize = bundledSize
+      larger = ''
+    }
+    else {
+      larger = ` (~${roundHalf(bundledSize / awsLiteBundledSize)}x larger bundle size)`
+    }
+    console.log(`- Bundled size: ${formatSize(bundledSize)}${larger}`)
+    console.log('')
+  }
+}
+main()

bench/benchmark-speed.js

@@ -0,0 +1,134 @@
+#! /usr/bin/env node
+
+let { join } = require('path')
+let { execSync }  = require('child_process')
+let { readFileSync } = require('fs')
+let percentile = require('percentile')
+let { formatSize, roundHalf } = require('./_helpers')
+
+let benchmarkRuns = process.env.AWS_LITE_BENCHMARK_RUNS || 50
+let harness = readFileSync(join(__dirname, '_harness.js')).toString()
+
+let benchmarks = {
+  'aws-lite': {
+    import: {
+      script2: `require('../');`,
+      runs: [],
+    },
+    'import & instantiate': {
+      script1: ``,
+      script2: `let awsLite = require('../'); let client = awsLite({ region: 'us-west-1' });`,
+      runs: [],
+    },
+  },
+  'AWS SDK v2': {
+    import: {
+      script2: `
+        process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = true;
+        require('aws-sdk');
+      `,
+      runs: [],
+    },
+    'import & instantiate': {
+      script2: `
+        process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = true;
+        let Aws = require('aws-sdk');
+        let aws = new Aws.DynamoDB({ region: 'us-west-1' });
+      `,
+      runs: [],
+    },
+  },
+  'AWS SDK v2 (single client)': {
+    import: {
+      script2: `
+        process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = true;
+        require('aws-sdk/clients/dynamodb');
+      `,
+      runs: [],
+    },
+    'import & instantiate': {
+      script2: `
+        process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = true;
+        let DynamoDB = require('aws-sdk/clients/dynamodb');
+        let aws = new DynamoDB({ region: 'us-west-1' });
+      `,
+      runs: [],
+    },
+  },
+  'AWS SDK v3': {
+    import: {
+      script2: `require('@aws-sdk/client-dynamodb');`,
+      runs: [],
+    },
+    'import & instantiate': {
+      script2: `
+        let DynamoDB = require('@aws-sdk/client-dynamodb');
+        let cmd = new DynamoDB.GetItemCommand({ RequestItem: { 'foo': { Keys: ['bar'] } } });
+      `,
+      runs: [],
+    },
+  }
+}
+
+let benchmarksToRun = [
+  // 'import', // Fine, but doesn't really tell us as much import & instantiate
+  'import & instantiate',
+  // TODO: round trip request to DynamoDB
+]
+
+let awsLiteAvgSpeed
+let awsLiteP95Speed
+let awsLiteAvgMemory
+let awsLiteP95Memory
+let slowerAvg = '', slowerP95 = '', largerAvg = '', largerP95 = ''
+
+for (let running of benchmarksToRun) {
+  console.log(`---------- Running ${running} benchmarks ---------- `)
+
+  for (let name of Object.keys(benchmarks)) {
+    let isAWSLite = name === 'aws-lite'
+    console.log(`[${name}]`)
+
+    console.log(`Starting ${running} benchmark`)
+    let { script1 = '', script2 } = benchmarks[name][running]
+    let start = Date.now()
+    for (let i = 0; i < benchmarkRuns; i++) {
+      let run = harness
+        .replace('/* $script1 */', script1)
+        .replace('/* $script2 */', script2)
+        .split('\n').join(' ')
+      let { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = process.env
+      let result = execSync(`node -e "${run}"`, { env: { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } })
+      benchmarks[name][running].runs.push(JSON.parse(result))
+    }
+    console.log(`Completed ${benchmarkRuns} runs in ${Date.now() - start} ms`)
+    let times = benchmarks[name][running].runs.map(run => run.times.result)
+    let avgTime = times.reduce((a, b) => a + b, 0) / times.length
+    let P95Time = percentile(95, times)
+    if (!isAWSLite) {
+      slowerAvg = ` (~${roundHalf(avgTime / awsLiteAvgSpeed)}x slower ${running})`
+      slowerP95 = ` (~${roundHalf(P95Time / awsLiteP95Speed)}x slower ${running})`
+    }
+    else {
+      awsLiteAvgSpeed = avgTime
+      awsLiteP95Speed = P95Time
+    }
+    console.log(`- Avg time to complete: ${avgTime} ms${slowerAvg}`)
+    console.log(`- p95 time to complete: ${P95Time} ms${slowerP95}`)
+
+    let memory = benchmarks[name][running].runs.map(run => run.memory.result)
+    let avgMemory = memory.reduce((a, b) => a + b, 0) / times.length
+    let P95Memory = percentile(95, memory)
+    if (!isAWSLite) {
+      largerAvg = ` (~${roundHalf(avgMemory / awsLiteAvgMemory)}x more memory)`
+      largerP95 = ` (~${roundHalf(P95Memory / awsLiteP95Memory)}x more memory)`
+    }
+    else {
+      awsLiteAvgMemory = avgMemory
+      awsLiteP95Memory = P95Memory
+    }
+    console.log(`- Avg memory footprint: ${formatSize(avgMemory)}${largerAvg}`)
+    console.log(`- p95 memory footprint: ${formatSize(P95Memory)}${largerP95}`)
+    console.log(``)
+  }
+}

bench/create-resources.js

@@ -0,0 +1,49 @@
+let awsLite = require('../')
+
+let service = 'dynamodb'
+let region = process.env.AWS_REGION || 'us-west-2'
+let TableName = process.env.AWS_LITE_TEST_TABLE_NAME || 'aws-lite-test-table'
+
+async function main () {
+  let aws = await awsLite({ region })
+
+  try {
+    let table = await aws({
+      service,
+      headers: { 'X-Amz-Target': 'DynamoDB_20120810.DescribeTable' },
+      payload: { TableName },
+    })
+    console.log(`Found test table!`, table)
+    return
+  }
+  catch (err) {
+    if (!err?.__type?.includes('ResourceNotFoundException')) {
+      throw err
+    }
+    console.log('Test table not found, creating table')
+  }
+
+  let table = await aws({
+    service,
+    headers: { 'X-Amz-Target': 'DynamoDB_20120810.CreateTable' },
+    payload: {
+      TableName,
+      KeySchema: [
+        {
+          AttributeName: 'id',
+          KeyType: 'HASH'
+        }
+      ],
+      AttributeDefinitions: [
+        {
+          AttributeName: 'id',
+          AttributeType: 'S',
+        }
+      ],
+      BillingMode: 'PAY_PER_REQUEST',
+    },
+  })
+  console.log(`Created new table: ${TableName} in ${region}`)
+  console.log(table)
+}
+main()

bench/entry-files/aws-lite.js

@@ -0,0 +1 @@
+require('../../')

bench/entry-files/aws-sdk-v2-client.js

@@ -0,0 +1 @@
+require('aws-sdk/clients/dynamodb')

bench/entry-files/aws-sdk-v2.js

@@ -0,0 +1 @@
+require('aws-sdk')

bench/entry-files/aws-sdk-v3.js

@@ -0,0 +1 @@
+require('@aws-sdk/client-dynamodb')