Choices. Compare. Version.

README.md

@@ -34,11 +34,35 @@ benchmark [arguments (optional)]
 * `-h`: Help on how to use the tool.
 * `compare`: Get comparative data for your benchmarks.
 
-# Benchmarks
-
-__Machine:__ MacBook Pro (Retina, 13-inch, Early 2015) **Processor**: 2,7 GHz Intel Core i5 **Memory**: 8 GB 1867 MHz DDR3
+> You may also compare all test results, at once, in a single table; `benchmark compare -t`
 
-__Method:__ `autocannon -c 10 -d 2 -p 1 localhost:3000` * 2, taking the second average
+# Benchmarks
 
+* __Machine:__ MacBook Pro (Late 2013 | 2.3 GHz Intel Core i7 | 16 GB 1600 MHz DDR3)
+* __Method:__ `autocannon -c 100 -d 5 -p 10 localhost:3000` (two rounds; one warm-up, one to measure).
+* __Node:__ `8.11.2`
 
-[![asciicast](https://asciinema.org/a/SA9z6g2HjOr0n80KwEvUaWoSS.png)](https://asciinema.org/a/SA9z6g2HjOr0n80KwEvUaWoSS)
+                    | Version | Router | Requests/s | Latency | Throughput/Mb
+------------------------ | ------- | :----: | ---------: | ------: | ------------:
+bare                     | 8.11.2  | ✗      | 28411.2    | 3.44    | 4.04
+connect-router           | 1.3.2   | ✓      | 30328      | 3.22    | 4.36
+connect                  | 3.6.6   | ✗      | 32076.8    | 3.04    | 4.53
+express-route-prefix     | 4.16.2  | ✓      | 14447.2    | 6.81    | 4.97
+express-with-middlewares | 4.16.2  | ✓      | 14988      | 6.53    | 5.42
+express                  | 4.16.2  | ✓      | 20776      | 4.7     | 3.24
+fastify-big-json         | 1.5.0   | ✓      | 5002.8     | 19.43   | 57.40
+**fastify**              | **1.5.0** | ✓    | **33924.81** | **2.87** | **5.33**
+hapi                     | 17.5.0  | ✓      | 20116.8    | 4.88    | 3.11
+koa-router               | 7.4.0   | ✓      | 22059.2    | 4.44    | 3.44
+koa                      | 2.5.0   | ✗      | 25364.8    | 3.86    | 3.99
+micro                    | 9.1.0   | ✗      | 31902.4    | 3.06    | 4.97
+microrouter              | 3.1.1   | ✓      | 17989.6    | 5.44    | 2.84
+polka                    | 0.4.0   | ✓      | 34177.6    | 2.85    | 3.79
+rayo                     | 0.5.8   | ✓      | 34238.4    | 2.85    | 3.84
+restify                  | 7.1.0   | ✓      | 23956.8    | 3.89    | 3.84
+spirit-router            | 0.5.0   | ✓      | 29272      | 3.16    | 4.58
+spirit                   | 0.6.1   | ✗      | 30785.6    | 3.01    | 4.83
+take-five                | 1.3.4   | ✓      | 24491.2    | 3.7     | 8.07
+total.js                 | 2.9.4   | ✓      | 21019.2    | 4.65    | 6.03
+trek-engine              | 1.0.5   | ✗      | 28718.4    | 3.4     | 4.06
+trek-router              | 1.2.0   | ✓      | 28513.6    | 3.43    | 3.99

benchmark-bench.js

@@ -3,6 +3,7 @@
 
 const inquirer = require('inquirer')
 const bench = require('./lib/bench')
+const { choices, list } = require('./lib/packages')
 
 function select (callback) {
   inquirer.prompt([
@@ -12,74 +13,9 @@ function select (callback) {
       name: 'list',
       choices: [
         new inquirer.Separator(' = The usual ='),
-        {
-          name: 'fastify',
-          checked: true
-        },
-        {
-          name: 'connect'
-        },
-        {
-          name: 'express'
-        },
-        {
-          name: 'hapi'
-        },
-        {
-          name: 'koa'
-        },
-        {
-          name: 'polka'
-        },
-        {
-          name: 'rayo'
-        },
-        {
-          name: 'restify'
-        },
-        {
-          name: 'take-five'
-        },
-        {
-          name: 'total.js'
-        },
-        {
-          name: '@leizm-web'
-        },
+        ...list(),
         new inquirer.Separator(' = The extras = '),
-        {
-          name: 'connect-router'
-        },
-        {
-          name: 'express-route-prefix'
-        },
-        {
-          name: 'express-with-middlewares'
-        },
-        {
-          name: 'fastify-big-json'
-        },
-        {
-          name: 'koa-router'
-        },
-        {
-          name: 'micro'
-        },
-        {
-          name: 'micro-router'
-        },
-        {
-          name: 'spirit'
-        },
-        {
-          name: 'spirit-router'
-        },
-        {
-          name: 'trek-engine'
-        },
-        {
-          name: 'trek-engine-router'
-        }
+        ...list(true)
       ],
       validate: function (answer) {
         if (answer.length < 1) {
@@ -135,28 +71,6 @@ inquirer.prompt([
   if (!opts.all) {
     select(list => bench(opts, list))
   } else {
-    bench(opts, [
-      'bare',
-      'connect',
-      'connect-router',
-      'express',
-      'express-route-prefix',
-      'express-with-middlewares',
-      'fastify',
-      'hapi',
-      'koa',
-      'koa-router',
-      'micro',
-      'micro-router',
-      'polka',
-      'rayo',
-      'restify',
-      'spirit',
-      'spirit-router',
-      'take-five',
-      'total.js',
-      'trek-engine',
-      'trek-engine-router'
-    ])
+    bench(opts, choices)
   }
 })

benchmark-compare.js

@@ -1,41 +1,45 @@
 #!/usr/bin/env node
 'use strict'
 
-const { existsSync } = require('fs')
 const inquirer = require('inquirer')
-const path = require('path')
 const chalk = require('chalk')
+const Table = require('cli-table')
+const { join } = require('path')
+const { readdirSync, readFileSync } = require('fs')
 const { compare } = require('./lib/autocannon')
+const { info } = require('./lib/packages')
 
-let choices = [
-  'bare',
-  'connect',
-  'connect-router',
-  'express',
-  'express-route-prefix',
-  'express-with-middlewares',
-  'fastify',
-  'hapi',
-  'koa',
-  'koa-router',
-  'micro',
-  'micro-router',
-  'polka',
-  'rayo',
-  'restify',
-  'spirit',
-  'spirit-router',
-  'take-five',
-  'total.js',
-  'trek-engine',
-  'trek-engine-router'
-]
+const resultsPath = join(process.cwd(), 'results')
+let choices = readdirSync(resultsPath)
+  .filter((file) => file.match(/(.+)\.json$/))
+  .sort()
+  .map((choice) => choice.replace('.json', ''))
 
-const resultsDirectory = path.join(process.cwd(), 'results')
-choices = choices.filter(choice => existsSync(path.join(resultsDirectory, `${choice}.json`)))
+const showAsTable = process.argv[2] === '-t'
+if (!choices.length) {
+  console.log(chalk.red('Benchmark to gather some results to compare.'))
+} else if (showAsTable) {
+  const table = new Table({
+    head: ['', 'Version', 'Router', 'Requests/s', 'Latency', 'Throughput/Mb']
+  })
+
+  const bold = (writeBold, str) => writeBold ? chalk.bold(str) : str
+  choices.forEach((result) => {
+    let data = readFileSync(`${resultsPath}/${result}.json`)
+    data = JSON.parse(data.toString())
+    const beBold = result === 'fastify'
+    const { version = 'N/A', hasRouter = false } = info(result) || {}
+    table.push([
+      bold(beBold, chalk.blue(result)),
+      bold(beBold, version),
+      bold(beBold, hasRouter ? '✓' : '✗'),
+      bold(beBold, data.requests.average),
+      bold(beBold, data.latency.average),
+      bold(beBold, (data.throughput.average / 1024 / 1024).toFixed(2))
+    ])
+  })
 
-if (choices.length === 0) {
-  console.log(chalk.red('Run benchmark first to gather results to compare.'))
+  console.log(table.toString())
 } else {
   inquirer.prompt([{
     type: 'list',

benchmark.js

@@ -4,6 +4,6 @@
 const program = require('commander')
 
 program
-  .command('bench', 'bench one or all packages', { isDefault: true })
-  .command('compare', 'compare twice package')
+  .command('bench', 'Benchmark one, multiple or all modules.', { isDefault: true })
+  .command('compare', 'Compare results by module.')
   .parse(process.argv)

lib/packages.js

@@ -0,0 +1,51 @@
+'use strict'
+
+const { dependencies } = require('../package')
+
+const packages = {
+  'bare': { version: '8.11.2' },
+  'connect': {},
+  'connect-router': { extra: true, package: 'router', hasRouter: true },
+  'express': { hasRouter: true },
+  'express-route-prefix': { extra: true, package: 'express', hasRouter: true },
+  'express-with-middlewares': { extra: true, package: 'express', hasRouter: true },
+  'fastify': { checked: true, hasRouter: true },
+  'fastify-big-json': { extra: true, package: 'fastify', hasRouter: true },
+  'hapi': { hasRouter: true },
+  'koa': {},
+  'koa-router': { extra: true, hasRouter: true },
+  'micro': { extra: true },
+  'microrouter': { extra: true, hasRouter: true },
+  'polka': { hasRouter: true },
+  'rayo': { hasRouter: true },
+  'restify': { hasRouter: true },
+  'spirit': { extra: true },
+  'spirit-router': { extra: true, hasRouter: true },
+  'take-five': { hasRouter: true },
+  'total.js': { hasRouter: true },
+  'trek-engine': { extra: true },
+  'trek-router': { extra: true, hasRouter: true }
+}
+
+let choices = []
+Object.keys(packages).forEach((pkg) => {
+  if (!packages[pkg].version) {
+    const version = dependencies[pkg] || dependencies[packages[pkg].package] || ''
+    packages[pkg].version = version.replace('^', '')
+  }
+  choices.push(pkg)
+})
+
+module.exports = {
+  choices: choices.sort(),
+
+  list: (extra = false) => {
+    return choices.map((c) => {
+      return extra === !!packages[c].extra ? Object.assign({}, packages[c], { name: c }) : null
+    }).filter(c => c)
+  },
+
+  info: (module) => {
+    return packages[module]
+  }
+}

package.json

@@ -1,11 +1,11 @@
 {
   "name": "fastify-benchmarks",
-  "version": "0.3.4",
+  "version": "0.3.6",
   "description": "Benchmarks for Fastify, a fast and low-overhead web framework.",
   "main": "index.js",
   "scripts": {
     "start": "node benchmark.js",
-    "compare": "node benchmark.js compare",
+    "compare": "node benchmark.js compare --",
     "test": "standard | snazzy",
     "standard": "standard | snazzy"
   },
@@ -20,26 +20,27 @@
     "autocannon": "^2.0.0",
     "autocannon-compare": "^0.3.0",
     "chalk": "^2.3.2",
+    "cli-table": "^0.3.1",
     "commander": "^2.14.1",
     "connect": "^3.6.6",
     "cors": "^2.8.4",
     "dns-prefetch-control": "^0.1.0",
     "express": "^4.16.2",
-    "fastify": "^1.0.0-rc.3",
+    "fastify": "^1.5.0",
     "frameguard": "^3.0.0",
-    "hapi": "^17.2.1",
+    "hapi": "^17.5.0",
     "hide-powered-by": "^1.0.0",
     "hsts": "^2.1.0",
     "ienoopen": "^1.0.0",
-    "inquirer": "^3.3.0",
+    "inquirer": "^5.2.0",
     "koa": "^2.5.0",
     "koa-router": "^7.4.0",
     "make-promises-safe": "^1.1.0",
     "micro": "^9.1.0",
     "microrouter": "^3.1.1",
-    "ora": "^1.4.0",
-    "polka": "^0.3.4",
-    "rayo": "^0.5.1",
+    "ora": "^2.1.0",
+    "polka": "^0.4.0",
+    "rayo": "^0.5.8",
     "restify": "^7.1.0",
     "router": "^1.3.2",
     "spirit": "^0.6.1",