Merge branch 'master' into getCodeActions

benchmarks/index.js

@@ -10,16 +10,13 @@ class Runner {
     constructor(options) {
         this.versions = options.versions;
         this.targets = options.targets;
-        this.iterations = options.iterations;
         this.noprepare = options.noprepare;
         this.project = options.project;
+        this.quick = options.quick;
     }
     run() {
         this.downloadFiles();
 
-        console.log('benchmark: Clearing previous benchmark results');
-        fsExtra.outputFileSync(path.join(__dirname, 'results.json'), '{}');
-
         if (!this.noprepare) {
             this.prepare();
         }
@@ -114,58 +111,29 @@ class Runner {
     }
 
     runBenchmarks() {
+        console.log('benchmark: Running benchmarks: \n');
+        const maxVersionLength = this.versions.reduce((acc, curr) => {
+            return curr.length > acc ? curr.length : acc;
+        }, 0);
+
+        const maxTargetLength = this.targets.reduce((acc, curr) => {
+            return curr.length > acc ? curr.length : acc;
+        }, 0);
+
         //run one target at a time
         for (const target of this.targets) {
             //run each of the versions within this target
             for (let versionIndex = 0; versionIndex < this.versions.length; versionIndex++) {
                 const version = this.versions[versionIndex];
-                //run the same test several times and take an average
-                for (let iteration = 0; iteration < this.iterations; iteration++) {
-                    readline.clearLine(process.stdout);
-                    readline.cursorTo(process.stdout, 0);
-                    const opsPerSecondText = iteration > 0 ? ` (${this.getLatestOpsPerSecond(target, version).toFixed(3).toLocaleString('en')} ops/sec)` : '';
-                    process.stdout.write(`Benchmarking ${target}@${version} (${iteration + 1} of ${this.iterations})${opsPerSecondText}`);
-
-                    execSync(`node target-runner.js "${version}" "${target}" "brighterscript${versionIndex + 1}" "${this.project}"`, {
-                        cwd: path.join(__dirname),
-                        stdio: 'inherit'
-                    });
-                }
-                readline.clearLine(process.stdout);
-                readline.cursorTo(process.stdout, 0);
-
-                process.stdout.write(`Benchmarking ${target}@${version} (done)`);
-            }
-            readline.clearLine(process.stdout);
-            readline.cursorTo(process.stdout, 0);
-            //log the final results to the console
-            this.logTargetResults(target);
-            process.stdout.write('\n');
-        }
-    }
+                process.stdout.write(`Benchmarking ${target}@${version}`);
 
-    /**
-     * Walk through the result list, and find the most recent result added
-     */
-    getLatestOpsPerSecond(target, version) {
-        const results = fsExtra.readJsonSync(path.join(__dirname, 'results.json'));
-        const targetData = results[target];
-        if (targetData) {
-            const versionData = targetData[version];
-            if (versionData && versionData.length > 0) {
-                return versionData[versionData.length - 1];
+                execSync(`node target-runner.js "${version}" "${maxVersionLength}" "${target}" "${maxTargetLength}" "brighterscript${versionIndex + 1}" "${this.project}" "${this.quick}"`, {
+                    cwd: path.join(__dirname),
+                    stdio: 'inherit'
+                });
             }
-        }
-    }
-
-    logTargetResults(target) {
-        const results = fsExtra.readJsonSync(path.join(__dirname, 'results.json'));
-        for (let version of this.versions) {
-            const versionResults = results[target][version];
-            const average = versionResults.reduce((a, b) => {
-                return a + b;
-            }, 0) / versionResults.length;
-            console.log(`${target}@${version} x ${average.toFixed(3).toLocaleString('en')} ops/sec`);
+            //print a newline to separate the targets
+            console.log('');
         }
     }
 }
@@ -187,20 +155,22 @@ let options = yargs
         description: 'Which benchmark targets should be run',
         defaultDescription: JSON.stringify(targets)
     })
-    .option('iterations', {
-        type: 'number',
-        description: 'The number of times the test should be run.',
-        default: 3
-    })
     .option('noprepare', {
         type: 'boolean',
+        alias: 'noinstall',
         description: 'Skip running npm install. Use this to speed up subsequent runs of the same test',
         default: false
     })
     .option('project', {
         type: 'string',
         description: 'File path to a project that should be used for complex benchmarking (like validation). If omitted, the tool will download and use https://github.com/chtaylo2/Roku-GooglePhotos'
     })
+    .option('quick', {
+        type: 'boolean',
+        alias: 'fast',
+        description: 'run a quick benchmark rather than the lower more precise version',
+        default: false
+    })
     .strict()
     .check(argv => {
         const idx = argv.versions.indexOf('latest');

benchmarks/target-runner.js

@@ -1,44 +1,64 @@
 const path = require('path');
 const fsExtra = require('fs-extra');
-const { Suite } = require('benchmark');
+const { Suite, formatNumber } = require('benchmark');
+const readline = require('readline');
+const chalk = require('chalk');
 
-const version = process.argv[2];
-const target = process.argv[3];
-const bscAlias = process.argv[4];
-const projectPath = process.argv[5];
+let idx = 2;
+
+const version = process.argv[idx++];
+const maxVersionLength = process.argv[idx++];
+const target = process.argv[idx++];
+const maxTargetLength = process.argv[idx++];
+const bscAlias = process.argv[idx++];
+const projectPath = process.argv[idx++];
+const quick = JSON.parse(process.argv[idx++]);
 
 const brighterscript = require(path.join(__dirname, 'node_modules', bscAlias));
 
 const addTargetTestFunction = require(path.join(__dirname, 'targets', target));
 (async () => {
-    const suite = new Suite('parser suite')
-        // add listeners
-        // .on('cycle', function (event) {
-        //     console.log(String(event.target));
-        // })
+    const suite = new Suite('parser suite', {
+        minSamples: 30000,
+        initCount: 30000,
+        minTime: 30000,
+        maxTime: -Infinity
+    })
+        .on('add', (event) => {
+            event.target.on('start cycle', function startCycle() {
+                const bench = this;
+                const size = bench.stats.sample.length;
+
+                if (!bench.aborted) {
+                    readline.clearLine(process.stdout);
+                    readline.cursorTo(process.stdout, 0);
+                    process.stdout.write(
+                        bench.name + ' x ' + formatNumber(bench.count) + ' (' +
+                        size + ' sample' + (size === 1 ? '' : 's') + ') ' + formatNumber(bench.hz.toFixed(3)) + ' ops/sec'
+                    );
+                }
+            });
+        })
         .on('error', (error) => {
             console.error(error.currentTarget[0].error || error);
         })
         .on('complete', function complete() {
-            const resultsPath = path.join(__dirname, 'results.json');
-            //write to results.json
-            const results = fsExtra.readJsonSync(resultsPath);
-            if (!results[target]) {
-                results[target] = {};
-            }
-            if (!results[target][version]) {
-                results[target][version] = [];
-            }
-            results[target][version].push(
-                //the ops/sec for the test run
-                this[0].hz
+            const hz = this[0].hz;
+            //write the final result to output
+            readline.clearLine(process.stdout);
+            readline.cursorTo(process.stdout, 0);
+            const formattedHz = formatNumber(hz.toFixed(3));
+            console.log(
+                `${target.padStart(maxTargetLength, ' ')}@${version.padEnd(maxVersionLength, ' ')}`,
+                '-'.repeat(' ###,###,###.###'.length - formattedHz.length),
+                chalk.yellow(formattedHz), 'ops/sec'
             );
-            fsExtra.outputFileSync(resultsPath, JSON.stringify(results, null, 4));
         });
-
     //add the test method. This could be async.
     await Promise.resolve(
-        addTargetTestFunction(suite, `${target}@${version}`, brighterscript, projectPath)
+        addTargetTestFunction(suite, `${target}@${version}`, brighterscript, projectPath, {
+            minTime: quick ? undefined : 3.5
+        })
     );
 
     suite.run({ 'async': true });

benchmarks/targets/lex-parse-validate.js

@@ -1,4 +1,4 @@
-module.exports = (suite, name, brighterscript, projectPath) => {
+module.exports = (suite, name, brighterscript, projectPath, options) => {
     const { ProgramBuilder } = brighterscript;
 
     suite.add(name, (deferred) => {
@@ -22,6 +22,7 @@ module.exports = (suite, name, brighterscript, projectPath) => {
             console.error(error);
         });
     }, {
+        ...options,
         'defer': true
     });
 };

benchmarks/targets/lexer.js → benchmarks/targets/lex.js

@@ -1,4 +1,4 @@
-module.exports = async (suite, name, brighterscript, projectPath) => {
+module.exports = async (suite, name, brighterscript, projectPath, options) => {
     const { ProgramBuilder, Lexer } = brighterscript;
 
     const builder = new ProgramBuilder();
@@ -20,5 +20,5 @@ module.exports = async (suite, name, brighterscript, projectPath) => {
         for (let brsFile of brsFiles) {
             Lexer.scan(brsFile.fileContents);
         }
-    });
+    }, options);
 };

benchmarks/targets/parse-brs.js

@@ -0,0 +1,33 @@
+module.exports = async (suite, name, brighterscript, projectPath, options) => {
+    const { ProgramBuilder } = brighterscript;
+
+    const builder = new ProgramBuilder();
+    //run the first run
+    await builder.run({
+        cwd: projectPath,
+        createPackage: false,
+        copyToStaging: false,
+        //disable diagnostic reporting (they still get collected)
+        diagnosticFilters: ['**/*'],
+        logLevel: 'error'
+    });
+    //collect all the XML files
+    const files = Object.values(builder.program.files).filter(x => ['.brs', '.bs', '.d.bs'].includes(x.extension));
+    if (files.length === 0) {
+        console.log('[parse-brs] No brs files found in program');
+        return;
+    }
+    suite.add(name, (deferred) => {
+        const promises = [];
+        for (const file of files) {
+            promises.push(
+                builder.program.addOrReplaceFile(file.pkgPath, file.fileContents)
+            );
+        }
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
+        Promise.all(promises).then(() => deferred.resolve());
+    }, {
+        ...options,
+        'defer': true
+    });
+};

benchmarks/targets/xml-parser.js → benchmarks/targets/parse-xml.js

@@ -1,4 +1,4 @@
-module.exports = async (suite, name, brighterscript, projectPath) => {
+module.exports = async (suite, name, brighterscript, projectPath, options) => {
     const { ProgramBuilder, XmlFile } = brighterscript;
 
     const builder = new ProgramBuilder();
@@ -34,6 +34,7 @@ module.exports = async (suite, name, brighterscript, projectPath) => {
             deferred.resolve();
         }
     }, {
+        ...options,
         'defer': true
     });
 };

benchmarks/targets/parser.js → benchmarks/targets/parse.js

@@ -1,4 +1,4 @@
-module.exports = async (suite, name, brighterscript, projectPath) => {
+module.exports = async (suite, name, brighterscript, projectPath, options) => {
     const { ProgramBuilder, Parser } = brighterscript;
 
     const builder = new ProgramBuilder();
@@ -21,5 +21,5 @@ module.exports = async (suite, name, brighterscript, projectPath) => {
         for (let brsFile of brsFiles) {
             Parser.parse(brsFile.parser.tokens);
         }
-    });
+    }, options);
 };

benchmarks/targets/transpile-brs.js

@@ -0,0 +1,31 @@
+module.exports = async (suite, name, brighterscript, projectPath, options) => {
+    const { ProgramBuilder } = brighterscript;
+
+    const builder = new ProgramBuilder();
+    //run the first run
+    await builder.run({
+        cwd: projectPath,
+        createPackage: false,
+        copyToStaging: false,
+        //disable diagnostic reporting (they still get collected)
+        diagnosticFilters: ['**/*'],
+        logLevel: 'error'
+    });
+    //collect all the brs files
+    const files = Object.values(builder.program.files).filter(x => ['.brs', '.bs'].includes(x.extension));
+
+    //flag every file for transpilation
+    for (const file of files) {
+        file.needsTranspiled = true;
+    }
+
+    if (files.length === 0) {
+        console.log('[transpile-brs] No brs|bs|d.bs files found in program');
+        return;
+    }
+    suite.add(name, () => {
+        for (const x of files) {
+            x.transpile();
+        }
+    }, options);
+};

benchmarks/targets/xml-transpile.js → benchmarks/targets/transpile-xml.js

@@ -1,4 +1,4 @@
-module.exports = async (suite, name, brighterscript, projectPath) => {
+module.exports = async (suite, name, brighterscript, projectPath, options) => {
     const { ProgramBuilder } = brighterscript;
 
     const builder = new ProgramBuilder();
@@ -12,14 +12,18 @@ module.exports = async (suite, name, brighterscript, projectPath) => {
         logLevel: 'error'
     });
     //collect all the XML files
-    const xmlFiles = Object.values(builder.program.files).filter(x => x.extension === '.xml');
-    if (xmlFiles.length === 0) {
+    const files = Object.values(builder.program.files).filter(x => x.extension === '.xml');
+    //flag every file for transpilation
+    for (const file of files) {
+        file.needsTranspiled = true;
+    }
+    if (files.length === 0) {
         console.log('[xml-transpile] No XML files found in program');
         return;
     }
     suite.add(name, () => {
-        for (const x of xmlFiles) {
+        for (const x of files) {
             x.transpile();
         }
-    });
+    }, options);
 };

benchmarks/targets/transpile.js

@@ -0,0 +1,30 @@
+module.exports = async (suite, name, brighterscript, projectPath, options) => {
+    const { ProgramBuilder } = brighterscript;
+
+    const builder = new ProgramBuilder();
+    //run the first run outside of the test
+    await builder.run({
+        cwd: projectPath,
+        createPackage: false,
+        copyToStaging: false,
+        //disable diagnostic reporting (they still get collected)
+        diagnosticFilters: ['**/*'],
+        logLevel: 'error'
+    });
+    if (Object.keys(builder.program.files).length === 0) {
+        throw new Error('No files found in program');
+    }
+
+    const files = Object.values(builder.program.files);
+
+    //force transpile for every file
+    for (const file of files) {
+        file.needsTranspiled = true;
+    }
+
+    suite.add(name, () => {
+        for (const file of files) {
+            file.transpile();
+        }
+    }, options);
+};