feat: Allowing custom runner configs (#99)

* fix: Fixing agent protocol
* feat: Allowing custom configs for runners

examples/simple_benchmark/best.config.js

@@ -2,13 +2,19 @@ module.exports = {
     projectName: 'simple-benchmarks',
     benchmarkOnClient: true,
     useMacroTaskAfterBenchmark: false,
-    benchmarkRunner: '@best/runner-headless',
-    // benchmarkRunner: '@best/runner-remote',
-
-    // This is for running @best/runner-remote
-    // "benchmarkRunnerConfig": {
-    //     "host": "http://localhost:5000",
-    //     "options": { path: '/best' },
-    //     "remoteRunner": "@best/runner-headless"
-    // },
+    "runnerConfig": [
+        {
+            "runner": "@best/runner-headless",
+            "name": "default"
+        },
+        {
+            "runner": '@best/runner-remote',
+            "name": "local",
+            "config": {
+                "host": "http://localhost:5000",
+                "options": { path: '/best' },
+                "remoteRunner": "@best/runner-headless"
+            }
+        }
+    ],
 };

examples/simple_lwc_benchmark/best.compat.config.js

@@ -8,12 +8,19 @@ module.exports = {
         }]
     ],
     benchmarkOnClient: false,
-    benchmarkRunner: '@best/runner-headless',
-
-    // This is for running @best/runner-remote
-    // "benchmarkRunnerConfig": {
-    //     "host": "http://localhost:5000",
-    //     "options": { path: '/best' },
-    //     "remoteRunner": "@best/runner-headless"
-    // },
+    "runnerConfig": [
+        {
+            "runner": "@best/runner-headless",
+            "name": "default"
+        },
+        {
+            "runner": '@best/runner-remote',
+            "name": "remote",
+            "config": {
+                "host": "http://localhost:5000",
+                "options": { path: '/best' },
+                "remoteRunner": "@best/runner-headless"
+            }
+        }
+    ],
 };

examples/simple_lwc_benchmark/best.prod.config.js

@@ -8,12 +8,19 @@ module.exports = {
         }]
     ],
     benchmarkOnClient: false,
-    benchmarkRunner: '@best/runner-headless',
-
-    // This is for running @best/runner-remote
-    // "benchmarkRunnerConfig": {
-    //     "host": "http://localhost:5000",
-    //     "options": { path: '/best' },
-    //     "remoteRunner": "@best/runner-headless"
-    // },
+    "runnerConfig": [
+        {
+            "runner": "@best/runner-headless",
+            "name": "default"
+        },
+        {
+            "runner": '@best/runner-remote',
+            "name": "remote",
+            "config": {
+                "host": "http://localhost:5000",
+                "options": { path: '/best' },
+                "remoteRunner": "@best/runner-headless"
+            }
+        }
+    ],
 };

packages/best-agent/src/broker.js

@@ -7,28 +7,37 @@ export default class Broker {
         this.clientQueue = [];
         this.runningTask = null;
         this.socketServer = socketServer;
-        socketServer.on('connect', socket => this.connectedClient(socket));
+        socketServer.on('connect', socket => this.setupConnection(socket));
     }
 
     isTaskRunning() {
         return !!this.runningTask;
     }
 
-    connectedClient(socket) {
+    setupConnection(socket) {
         const client = new SocketClient(socket);
         this.clients.set(client);
+        this.connectedClient(client);
+        client.on('disconnect', (reason) => this.disconnectedClient(client, reason));
+    }
 
+    connectedClient(client) {
         if (!this.isTaskRunning()) {
             this.runClientTask(client);
         } else {
             this.clientQueue.push(client);
-            client.setEnqueued();
+            client.setEnqueued({ pending: this.clientQueue.length });
         }
     }
 
+    disconnectedClient(client) {
+        this.clientQueue = this.clientQueue.filter(c => client !== c);
+    }
+
     runClientTask(client) {
         const task = new BenchmarkTask(client);
         task.on('complete', () => this.resetRunningTask());
+        task.on('error', () => this.resetRunningTask());
         this.runningTask = task;
         task.start();
     }

packages/best-agent/src/client.js

@@ -1,7 +1,7 @@
 import SocketIOFile from 'socket.io-file';
 import EventEmitter from 'events';
 import path from 'path';
-import { BENCHMARK_TASK, DISCONNECT, LOAD_BENCHMARK } from './operations';
+import { ERROR, BENCHMARK_TASK, DISCONNECT, LOAD_BENCHMARK } from './operations';
 import { cacheDirectory } from '@best/utils';
 import { x as extractTar } from 'tar';
 
@@ -16,8 +16,8 @@ const STATE_COMPLETED = 'done';
 const LOADER_CONFIG = {
     uploadDir: path.join(cacheDirectory('best_agent'), 'uploads'),
     accepts: [],
-    maxFileSize: 4194304,
-    chunkSize: 10240, // 1kb
+    maxFileSize: 52428800, // 50 mb
+    chunkSize: 10240, // 10kb
     transmissionDelay: 0,
     overwrite: true,
 };
@@ -76,8 +76,8 @@ class SocketClient extends EventEmitter {
                 clearTimeout(this._timeout);
                 this.socket.disconnect(true);
             }
-            this.emit(DISCONNECT);
             this._log(`STATUS: disconnected (${forcedError || 'socket disconnected'})`);
+            this.emit(DISCONNECT);
         }
     }
 
@@ -106,17 +106,23 @@ class SocketClient extends EventEmitter {
         });
     }
 
+    onUploaderError(data) {
+        this.emit(ERROR, data);
+    }
+
     loadBenchmarks() {
         const uploader = new SocketIOFile(this.socket, LOADER_CONFIG);
         uploader.on('start', () => clearTimeout(this._timeout));
         uploader.on('stream', ({ wrote, size }) => this._log(`downloading ${wrote} / ${size}`));
         uploader.on('complete', info => this.onLoadedBenchmarks(info));
+        uploader.on('error', (err) => this.onUploaderError(err));
         this.setState(STATE_LOADING_FILES);
         this.socket.emit(LOAD_BENCHMARK);
         this.setTimeout(5000);
     }
 
-    setEnqueued() {
+    setEnqueued(status) {
+        this.socket.emit('benchmark_enqueued', status);
         this.setState(STATE_QUEUED);
     }
 
@@ -127,7 +133,7 @@ class SocketClient extends EventEmitter {
     sendBenchmarkResults(err, benchmarkResults) {
         if (err) {
             this._log(`Sending error`);
-            this.socket.emit('benchmark_error', err);
+            this.socket.emit('benchmark_error', err.toString());
         } else {
             this._log(`Sending results`);
             this.socket.emit('benchmark_results', benchmarkResults);

packages/best-agent/src/operations.js

@@ -1,3 +1,4 @@
 export const BENCHMARK_TASK = 'benchmark_task';
 export const DISCONNECT = 'disconnect';
 export const LOAD_BENCHMARK = 'load_benchmark';
+export const ERROR = 'error';

packages/best-agent/src/task.js

@@ -1,13 +1,13 @@
 import EventEmitter from 'events';
 import SocketClient from './client';
 import { runBenchmark } from '@best/runner';
+import { ERROR } from './operations';
 
 const WAITING_FOR_CONFIG = 'waiting_for_config';
 const WAITING_FOR_BENCHMARK = 'waiting_for_benchmark';
 const WAITING_FOR_ABORT = 'waiting_for_abort';
 const RUNNING = 'running';
 const COMPLETED = 'complete';
-const CLIENT_DISCONNECTED = 'client_disconnected';
 
 let counter = 0;
 export default class BenchmarkTask extends EventEmitter {
@@ -18,7 +18,8 @@ export default class BenchmarkTask extends EventEmitter {
         this.client = client;
         client.on(SocketClient.CONFIG_READY, config => this.onBenchmarkConfigReady(config));
         client.on(SocketClient.BENCHMARK_READY, () => this.onBenchmarksReady());
-        client.on('disconnect', () => this.onDisconnectedClient());
+        client.on('disconnect', () => this.onClientDisconnected());
+        client.on('error', (err) => this.onClientError(err));
     }
     _log(msg) {
         process.stdout.write(`Task[${this.id}] - ${msg}\n`);
@@ -32,14 +33,20 @@ export default class BenchmarkTask extends EventEmitter {
         }
     }
 
-    onDisconnectedClient() {
+    onClientDisconnected() {
         if (this.state === RUNNING) {
             this._log('Aborting task. Client disconnected while running the task');
             this._log('Benchmark will finish before releasing the task');
             this.state = WAITING_FOR_ABORT;
         }
     }
 
+    onClientError(err) {
+        this._log('Error running task:' + err.toString());
+        console.log('Stack trace: ', err);
+        this.emit(ERROR, err);
+    }
+
     onBenchmarkConfigReady() {
         this.start();
     }
@@ -52,28 +59,25 @@ export default class BenchmarkTask extends EventEmitter {
     async runBenchmarkTask(benchmarkConfig) {
         this.state = RUNNING;
         const { benchmarkName } = benchmarkConfig;
+        let results;
+        let error;
         try {
             this._log(`Running benchmark ${benchmarkName}`);
-            const results = await runBenchmark(benchmarkConfig, this.client.getMessager());
+            results = await runBenchmark(benchmarkConfig, this.client.getMessager());
             this._log(`Benchmark ${benchmarkName} completed successfully`);
-            this.afterRunBenchmark(results);
         } catch (err) {
             this._log(`Something went wrong while running ${benchmarkName}`);
             process.stderr.write(err + '\n');
-            this.client.sendBenchmarkResults(err);
+            error = err;
+        } finally {
+            this.afterRunBenchmark(error, results);
         }
     }
 
-    afterRunBenchmark(results) {
-        if (this.state === WAITING_FOR_ABORT) {
-            this.state = CLIENT_DISCONNECTED;
-            this.emit(COMPLETED);
-            return;
-        }
-
+    afterRunBenchmark(error, results) {
         this.state = COMPLETED;
         this._log(`Sending results to client`);
-        this.client.sendBenchmarkResults(null, results);
+        this.client.sendBenchmarkResults(error, results);
         this.emit(COMPLETED);
     }
 }

packages/best-cli/src/cli/args.js

@@ -58,4 +58,17 @@ export const options = {
         description: 'Integrates with Git, posting the results of the benchmark or comparison',
         type: 'boolean',
     },
+    runner: {
+        default: 'default',
+        description:
+            'Select the runner to execute the benchmarks.' +
+            'Make sure you have defined `runnerConfig` options in your Best config file' +
+            'By default it will use @best/runner-headless',
+        type: 'string',
+    },
+    runnerConfig: {
+        default: undefined,
+        description: 'JSON representation of the configuration to use for the give runner',
+        type: 'string',
+    },
 };

packages/best-config/src/defaults.js

@@ -7,8 +7,8 @@ export default {
     moduleFileExtensions: ['js'],
     moduleNameMapper: {},
     modulePathIgnorePatterns: [],
-    benchmarkRunner: '@best/runner-headless',
-    benchmarkRunnerConfig: {},
+    runner: "default",
+    runnerConfig: [{ runner: '@best/runner-headless', config: {} }],
     benchmarkEnvironment: 'production',
     benchmarkMaxDuration: 1000 * 10, // 10s
     benchmarkMinIterations: 30,

packages/best-config/src/index.js

@@ -155,6 +155,21 @@ function normalizePlugins(plugins, { rootDir }) {
     });
 }
 
+function normalizeRunnerConfig(runnerConfig, { runner }) {
+    if (!Array.isArray(runnerConfig)) {
+        runnerConfig = [runnerConfig];
+    }
+
+    const defaultRunners = runnerConfig.filter(c => c.name === undefined || c.name === 'default');
+    if (defaultRunners > 1) {
+        throw new Error('Wrong configuration: More than one default configuration declared');
+    }
+
+    const match = runnerConfig.find(c => c.name === runner) || defaultRunners[0] || {};
+
+    return match;
+}
+
 function normalizeCommits([base, target]) {
     base = base + '' || BASE_COMMIT || '';
     target = target + '' || TARGET_COMMIT || '';
@@ -173,6 +188,9 @@ function normalize(options, argsCLI) {
             case 'plugins':
                 value = normalizePlugins(options[key], options);
                 break;
+            case 'runnerConfig':
+                value = normalizeRunnerConfig(options[key], options);
+                break;
             case 'compareStats':
                 value = options[key].length ? normalizeCommits(options[key], options) : undefined;
                 break;
@@ -227,8 +245,8 @@ function _getConfigs(options) {
             roots: options.roots,
 
             projectName: options.projectName,
-            benchmarkRunner: options.benchmarkRunner,
-            benchmarkRunnerConfig: options.benchmarkRunnerConfig,
+            benchmarkRunner: options.runnerConfig.runner || options.runner,
+            benchmarkRunnerConfig: options.runnerConfig.config || options.runnerConfig,
             benchmarkEnvironment: options.benchmarkEnvironment,
             benchmarkEnvironmentOptions: options.benchmarkEnvironmentOptions,
             benchmarkMaxDuration: options.benchmarkMaxDuration,

packages/best-messager/src/messager-runner.js

@@ -170,6 +170,11 @@ export default class RunnerMessager {
         bench.state = BUILD_STATE.ERROR;
     }
 
+    logState(state) {
+        this._currentState = state;
+        this._update();
+    }
+
     finishRun() {
         this._update(true);
     }
@@ -230,6 +235,8 @@ export default class RunnerMessager {
             buffer += renderTime(progress.runtime, progress.estimated, 40);
         }
 
+        buffer += this._currentState ? `\n Status: ${this._currentState} \n` : '\n';
+
         if (errors) {
             buffer += '\n' + errors;
         }