feat(best-config): Enable multi-project configurations (#60)

* Create new example project
* Collect repository info
* Fixing duration on runner node + bubbling errors on benchmark construction
* Fix rootDir paths
* Fix linting
* Fix perf command
* Fix output
* Minor label change
* Change perf test folder

.circleci/config.yml

@@ -61,11 +61,11 @@ jobs:
 
       - run:
           name: Performance Test - Example project
-          command: yarn perf:example
+          command: yarn perf:lwc
 
       - run:
           name: Comparing perf examples
-          command: cd examples/simple_benchmark/ && best --compareStats ${BASE_COMMIT} ${TARGET_COMMIT} --externalStorage=@best/store-aws --gitIntegration
+          command: cd examples/simple_lwc_benchmark/ && best --compareStats ${BASE_COMMIT} ${TARGET_COMMIT} --externalStorage=@best/store-aws --gitIntegration
 
 workflows:
   version: 2

.eslintrc

@@ -7,7 +7,7 @@
         "node": true
     },
     "rules": {
-        "no-console": ["warn", { "allow": ["timeStamp", "time", "timeEnd"] }],
+        "no-console": ["warn", { "allow": ["timeStamp", "time", "timeEnd", "log", "warn", "error"] }],
         "no-await-in-loop": 0,
         "no-confusing-arrow": ["error", { "allowParens": true }],
         "eslint-comments/no-use": false,

best.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+    projects: [
+        '<rootDir>/examples/simple_benchmark',
+        '<rootDir>/examples/simple_lwc_benchmark',
+    ]
+};

examples/simple_benchmark/best.config.js

@@ -1,11 +1,7 @@
 module.exports = {
-    projectName: 'perf-best-example',
-    plugins: {
-        'rollup-plugin-lwc-compiler': {
-            rootDir: '<rootDir>/src/',
-        },
-    },
-    benchmarkOnClient: false,
+    projectName: 'perf-best-simple-example',
+    benchmarkOnClient: true,
+    useMacroTaskAfterBenchmark: false,
     benchmarkRunner: '@best/runner-headless',
 
     // This is for running @best/runner-remote

examples/simple_benchmark/package.json

@@ -7,12 +7,6 @@
     "perf": "best"
   },
   "devDependencies": {
-    "best": "0.0.7",
-    "cross-env": "^5.1.3",
-    "lwc-compiler": "0.17.6",
-    "lwc-engine": "0.17.6",
-    "rollup": "~0.53.0",
-    "rollup-plugin-lwc-compiler": "0.17.6",
-    "rollup-plugin-replace": "2.0.0"
+    "best": "0.0.7"
   }
 }

examples/simple_benchmark/src/__benchmarks__/mock_data.js

@@ -0,0 +1,23 @@
+export const MOCK_DATA_SMALL = {
+    "id": "0001",
+    "type": "donut",
+    "name": "Cake",
+    "ppu": 0.55,
+    "batters": {
+        "batter":[
+            { "id": "1001", "type": "Regular" },
+            { "id": "1002", "type": "Chocolate" },
+            { "id": "1003", "type": "Blueberry" },
+            { "id": "1004", "type": "Devil's Food" }
+        ]
+    },
+    "topping": [
+        { "id": "5001", "type": "None" },
+        { "id": "5002", "type": "Glazed" },
+        { "id": "5005", "type": "Sugar" },
+        { "id": "5007", "type": "Powdered Sugar" },
+        { "id": "5006", "type": "Chocolate with Sprinkles" },
+        { "id": "5003", "type": "Chocolate" },
+        { "id": "5004", "type": "Maple" }
+    ]
+};

examples/simple_benchmark/src/__benchmarks__/object_keys.benchmark.js

@@ -0,0 +1,15 @@
+import { MOCK_DATA_SMALL } from "./mock_data";
+
+describe('deep_clone', () => {
+    benchmark('stringify_parse', () => {
+        run(() => {
+            JSON.parse(JSON.stringify(MOCK_DATA_SMALL));
+        });
+    });
+
+    benchmark('object_assign', () => {
+        run(() => {
+            Object.assign({}, MOCK_DATA_SMALL);
+        });
+    });
+});

examples/simple_benchmark/src/main.js

@@ -1,6 +0,0 @@
-import { createElement } from 'engine';
-import App from 'simple-benchmark';
-
-const container = document.getElementById('main');
-const element = createElement('simple-benchmark', { is: App });
-container.appendChild(element);

examples/simple_lwc_benchmark/best.config.js

@@ -0,0 +1,17 @@
+module.exports = {
+    projectName: 'perf-best-example',
+    plugins: {
+        'rollup-plugin-lwc-compiler': {
+            rootDir: '<rootDir>/src/',
+        },
+    },
+    benchmarkOnClient: false,
+    benchmarkRunner: '@best/runner-headless',
+
+    // This is for running @best/runner-remote
+    // "benchmarkRunnerConfig": {
+    //     "host": "http://localhost:5000",
+    //     "options": { path: '/best' },
+    //     "remoteRunner": "@best/runner-headless"
+    // },
+};

examples/simple_lwc_benchmark/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@best/simple_lwc_benchmark",
+  "version": "0.0.7",
+  "private": true,
+  "scripts": {
+    "build": "cross-env rollup -c ./scripts/rollup.config.js",
+    "perf": "best"
+  },
+  "devDependencies": {
+    "best": "0.0.7",
+    "cross-env": "^5.1.3",
+    "lwc-compiler": "0.17.11",
+    "lwc-engine": "0.17.11",
+    "rollup": "~0.53.0",
+    "rollup-plugin-lwc-compiler": "0.17.11",
+    "rollup-plugin-replace": "2.0.0"
+  }
+}

examples/simple_lwc_benchmark/src/main.js

@@ -0,0 +1,6 @@
+import { createElement } from 'engine';
+import App from 'simple-benchmark';
+
+const container = document.getElementById('main');
+const element = createElement('simple-benchmark', { is: App });
+container.appendChild(element);

package.json

@@ -15,7 +15,7 @@
     "build": "node scripts/build.js",
     "watch": "yarn build && node ./scripts/watch.js",
     "test": "jest --config jest.config.json",
-    "perf:example": "cd examples/simple_benchmark && best --externalStorage=@best/store-aws",
+    "perf:lwc": "cd examples/simple_lwc_benchmark && best --externalStorage=@best/store-aws",
     "release": "yarn prepare && lerna publish --exact --force-publish=* --registry='https://npm.lwcjs.org' && yarn changelog",
     "release:internal": "yarn prepare && lerna publish --exact --force-publish=* --skip-git --repo-version `node -pe \"require('./lerna.json').version\"` --yes --registry='https://nexus.soma.salesforce.com/nexus/content/repositories/npmjs-internal/'",
     "changelog": "yarn changelog:generate && yarn changelog:publish",
@@ -45,6 +45,6 @@
   },
   "workspaces": [
     "packages/*",
-    "examples/simple_benchmark"
+    "examples/*"
   ]
 }

packages/best-agent/src/client.js

@@ -50,7 +50,7 @@ class SocketClient extends EventEmitter {
         this._log('STATUS: connected');
     }
     _log(msg) {
-        console.log(`Client[${this.socket.id}] - ${msg}`);
+        process.stdout.write(`Client[${this.socket.id}] - ${msg}\n`);
     }
 
     setTimeout(t) {

packages/best-agent/src/task.js

@@ -21,7 +21,7 @@ export default class BenchmarkTask extends EventEmitter {
         client.on('disconnect', () => this.onDisconnectedClient());
     }
     _log(msg) {
-        console.log(`Task[${this.id}] - ${msg}`);
+        process.stdout.write(`Task[${this.id}] - ${msg}\n`);
     }
 
     start() {
@@ -59,7 +59,7 @@ export default class BenchmarkTask extends EventEmitter {
             this.afterRunBenchmark(results);
         } catch (err) {
             this._log(`Something went wrong while running ${benchmarkName}`);
-            console.log(err);
+            process.stderr.write(err + '\n');
             this.client.sendBenchmarkResults(err);
         }
     }

packages/best-analyzer/src/stats.js

@@ -63,7 +63,7 @@ export function compare(sample1, sample2) {
     }
 
     function getZ(n) {
-        return (n - size1 * size2 / 2) / Math.sqrt(size1 * size2 * (size1 + size2 + 1) / 12);
+        return (n - (size1 * size2 / 2)) / Math.sqrt(size1 * size2 * (size1 + size2 + 1) / 12);
     }
 
     if (size1 + size2 < 30) {

packages/best-build/src/__tests__/best-build.spec.js

@@ -17,7 +17,7 @@ function tempDir() {
 describe('buildBenchmark', () => {
     test('generating index.js and index.html', async () => {
         const cacheDirectory = tempDir();
-        const res = await buildBenchmark(
+        await buildBenchmark(
             path.resolve(__dirname, 'fixtures', 'single-file', 'single-file.js'),
             {
                 cacheDirectory,
@@ -107,7 +107,7 @@ describe('buildBenchmark', () => {
         jest.doMock(
             'build-plugin-hooks',
             () => {
-                return options => {
+                return () => {
                     return {
                         load: id => {
                             loaded.push(id);

packages/best-build/src/index.js

@@ -2,7 +2,6 @@ import { rollup } from 'rollup';
 import path from 'path';
 import benchmarkRollup from './rollup-plugin-benchmark-import';
 import { generateDefaultHTML } from './html-templating';
-import { BuildStateMessager } from '@best/messager';
 import fs from 'fs';
 import crypto from 'crypto';
 

packages/best-build/src/rollup-plugin-benchmark-import.js

@@ -27,7 +27,7 @@ function resolveModuleEntryFromPackage(module) {
 const BENCHMARK_RUNTIME_MODULE = '@best/runtime';
 const BENCHMARK_IMPORT = `import { ${PRIMITIVES.join(',')} } from "${BENCHMARK_RUNTIME_MODULE}" \n`;
 
-export default function() {
+export default function () {
     let input;
     return {
         name: 'benchmark-import',

packages/best-cli/src/cli/args.js

@@ -18,7 +18,8 @@ export const options = {
     },
     projects: {
         description:
-            'A list of projects that use Best to run all tests of all ' + 'projects in a single instance of Best.',
+            'A list of projects that use Best to run all tests of all ' +
+            'projects in a single instance of Best.',
         type: 'array',
     },
     iterations: {

packages/best-cli/src/cli/index.js

@@ -40,7 +40,7 @@ export async function run(maybeArgv, project) {
     } catch (error) {
         const errParts = error.stack.split('\n');
         errorMessager.print(errParts.shift());
-        console.error(errParts.join('\n'));
+        console.warn(errParts.join('\n'));
         process.exit(1);
         throw error;
     }

packages/best-cli/src/cli/output.js

@@ -4,8 +4,8 @@ import chalk from 'chalk';
 function padding(n) {
     return n > 0
         ? Array.apply(null, Array((n - 1) * 3))
-              .map(() => ' ')
-              .join('') + '└─ '
+            .map(() => ' ')
+            .join('') + '└─ '
         : '';
 }
 
@@ -81,9 +81,8 @@ export function generateReportTables(results, stream) {
 function generateComparisonRows(table, stats, name = '') {
     return stats.comparison.map(node => {
         if (node.comparison) {
-            return generateComparisonRows(table, node, `${node.benchmarkName || node.name}:`).reduce((a, b) =>
-                a.concat(b),
-            );
+            const nodeName = `${node.benchmarkName || node.name}:`;
+            return generateComparisonRows(table, node, nodeName).reduce((a, b) => a.concat(b), []);
         }
 
         const durationMetric = node.metrics.duration;
@@ -95,17 +94,19 @@ function generateComparisonRows(table, stats, name = '') {
             `${targetStats.median.toFixed(2)} (± ${targetStats.medianAbsoluteDeviation.toFixed(2)}ms)`,
             samplesComparison === 0 ? 'SAME' : samplesComparison === 1 ? 'SLOWER' : 'FASTER',
         ]);
+
+        return [];
     });
 }
 
 export function generateComparisonTable(comparisonStats, stream) {
-    const { baseCommit, targetCommit, comparison } = comparisonStats;
+    const { baseCommit, targetCommit } = comparisonStats;
     const table = new Table({
-        head: ['Benchmark name', `base(${baseCommit})`, `base(${targetCommit})`, 'Trend'],
+        head: ['Benchmark name', `base(${baseCommit})`, `target(${targetCommit})`, 'Trend'],
         colWidths: [50, 20, 20, 10],
     });
-
+    console.log(JSON.stringify(comparisonStats, null, '  '));
     generateComparisonRows(table, comparisonStats);
 
-    console.log(table.toString());
+    stream.write(table.toString() + '\n');
 }

packages/best-cli/src/run_best.js

@@ -6,18 +6,12 @@ import { storeBenchmarkResults } from '@best/store';
 import { analyzeBenchmarks } from '@best/analyzer';
 import path from 'path';
 
-async function getBenchmarkPaths(globalConfig, config) {
-    const rootDir = globalConfig.rootDir;
-    if (globalConfig.nonFlagArgs && globalConfig.nonFlagArgs.length) {
-        const results = await globby(globalConfig.nonFlagArgs, { cwd: rootDir });
-        if (results && results.length) {
-            return results.map(p => path.resolve(rootDir, p));
-        }
-        throw new Error(`Couldn't find any tests based on globby expression given.` + results);
-    }
-    const { testMatch } = config;
-    const results = await globby(testMatch, { cwd: rootDir });
-    return results.map(p => path.resolve(rootDir, p));
+async function getBenchmarkPaths({ nonFlagArgs, rootDir }, config) {
+    const { testMatch, rootDir: projectRoot } = config;
+    const rootPath = projectRoot || rootDir;
+    const paths = nonFlagArgs && nonFlagArgs.length ? nonFlagArgs : testMatch;
+    const results = await globby(paths, { cwd: rootPath });
+    return results.map(p => path.resolve(rootPath, p));
 }
 
 async function getBenchmarkTests(configs, globalConfig) {

packages/best-config/src/git.js

@@ -39,12 +39,37 @@ function getBranch(cwd) {
     });
 }
 
+function getRepository(cwd) {
+    return new Promise((resolve, reject) => {
+        const args = ['ls-remote', '--get-url'];
+        const child = childProcess.spawn('git', args, { cwd });
+        let stdout = '';
+        let stderr = '';
+        child.stdout.on('data', data => (stdout += data));
+        child.stderr.on('data', data => (stderr += data));
+        child.on('error', e => reject(e));
+        child.on('close', code => {
+            if (code === 0) {
+                const rawValue = stdout.trim();
+                const [owner, repo] = rawValue.split(':')
+                    .pop()
+                    .split('.git')[0].split('/');
+                resolve({ owner, repo });
+            } else {
+                reject(code + ': ' + stderr);
+            }
+        });
+    });
+}
+
 export async function addGitInformation(options) {
     const cwd = options.rootDir;
-    const [hash, localChanges, branch] = await Promise.all([getCurrentHash(cwd), hasLocalChanges(cwd), getBranch(cwd)]);
+    const [gitCommit, gitLocalChanges, gitBranch, gitRepository] = await Promise.all([
+        getCurrentHash(cwd),
+        hasLocalChanges(cwd),
+        getBranch(cwd),
+        getRepository(cwd)
+    ]);
 
-    options.gitCommit = hash;
-    options.gitLocalChanges = localChanges;
-    options.gitBranch = branch;
-    return options;
+    return Object.assign(options, { gitCommit, gitLocalChanges, gitBranch, gitRepository });
 }