fix(ci): report build script failure + revert node test runner to QUnit

scripts/index.js

@@ -1,4 +1,18 @@
 #!/usr/bin/env node
+
+/**
+ * Dearest fabric maintainer 💗,
+ * This file contains the cli logic, which governs most of the available commands fabric has to offer.
+ * 
+ * 📢 **IMPORTANT**
+ * CI uses these commands.
+ * In order for CI to correctly report the result of the command, the process must receive a correct exit code
+ * meaning that if you `spawn` a process, make sure to listen to the `exit` event and terminate the main process with the relevant code.
+ * Failing to do so will make CI report a false positive 📉.
+ */
+
+
+
 const fs = require('fs-extra');
 const os = require('os');
 const _ = require('lodash');
@@ -121,13 +135,13 @@ inquirer.registerPrompt('test-selection', ICheckbox);
 
 
 function build(options = {}) {
-    const args = ['rollup', '-c', options.watch ? '--watch' : ''];
+    const cmd = ['rollup', '-c', options.watch ? '--watch' : ''].join(' ');
     let minDest;
     if (options.output && !options.fast) {
         const { name, base, ...rest } = path.parse(path.resolve(options.output));
         minDest = path.format({ name: `${name}.min`, ...rest });
     }
-    return cp.spawn(args.join(' '), {
+    const processOptions = {
         stdio: 'inherit',
         shell: true,
         cwd: wd,
@@ -138,7 +152,21 @@ function build(options = {}) {
             BUILD_OUTPUT: options.output,
             BUILD_MIN_OUTPUT: minDest
         },
-    });
+    }
+    if (options.watch) {
+        cp.spawn(cmd, processOptions);
+    }
+    else {
+        try {
+            cp.execSync(cmd, processOptions);
+        } catch (error) {
+            // minimal logging, no need for stack trace
+            console.error(error.message);
+            // inform ci
+            process.exit(1);
+        }
+    }
+
 }
 
 function startWebsite() {
@@ -244,30 +272,55 @@ function exportToWebsite(options) {
     })
 }
 
-
 /**
- *
- * @param {'unit' | 'visual'} suite
- * @param {string[] | null} tests file paths
- * @param {{debug?:boolean,recreate?:boolean,verbose?:boolean,filter?:string}} [options]
+ * 
+ * @returns {Promise<boolean | undefined>} true if some tests failed
  */
-async function test(suite, tests, options = {}) {
-    const port = options.port || suite === 'visual' ? 8081 : 8080;
+async function runTestem({ suite, port, launch, dev, processOptions, context } = {}) {
+    port = port || suite === 'visual' ? 8081 : 8080;
     try {
         await killPort(port);
     } catch (error) {
 
     }
 
-    const args = [
-        'testem',
-        !options.dev ? 'ci' : '',
+    if (launch) {
+        // open localhost
+        const url = `http://localhost:${port}/`;
+        const start = (os.platform() === 'darwin' ? 'open' : os.platform() === 'win32' ? 'start' : 'xdg-open');
+        cp.exec([start, url].join(' '));
+    }
+
+    const processCmdOptions = [
         '-p', port,
         '-f', `test/testem.${suite}.js`,
-        '-l', options.context.map(_.upperFirst).join(',')
+        '-l', context.map(_.upperFirst).join(',')
     ];
 
-    cp.spawn(args.join(' '), {
+    if (dev) {
+        cp.spawn(['testem', ...processCmdOptions].join(' '), {
+            ...processOptions,
+            detached: true
+        });
+    }
+    else {
+        try {
+            cp.execSync(['testem', 'ci', ...processCmdOptions].join(' '), processOptions);
+        } catch (error) {
+            return true;
+        }
+    }
+}
+
+/**
+ *
+ * @param {'unit' | 'visual'} suite
+ * @param {string[] | null} tests file paths
+ * @param {{debug?:boolean,recreate?:boolean,verbose?:boolean,filter?:string}} [options]
+ * @returns {Promise<boolean | undefined>} true if some tests failed
+ */
+async function test(suite, tests, options = {}) {
+    const processOptions = {
         cwd: wd,
         env: {
             ...process.env,
@@ -281,20 +334,31 @@ async function test(suite, tests, options = {}) {
         },
         shell: true,
         stdio: 'inherit',
-        detached: options.dev
-    })
-        .on('exit', (code) => {
-            // propagate failed exit code to the process for ci to fail
-            // don't exit if tests passed - this is for parallel local testing
-            code && process.exit(code);
-        });
+    }
 
-    if (options.launch) {
-        // open localhost
-        const url = `http://localhost:${port}/`;
-        const start = (os.platform() === 'darwin' ? 'open' : os.platform() === 'win32' ? 'start' : 'xdg-open');
-        cp.exec([start, url].join(' '));
+    let failed = false;
+
+    // temporary revert
+    // run node tests directly with qunit
+    if (options.context.includes('node')) {
+        try {
+            cp.execSync(processOptions.env.NODE_CMD, processOptions);            
+        } catch (error) {
+            failed = true;
+        }
+    }
+
+    const browserContexts = options.context.filter(c => c !== 'node');
+    if (browserContexts.length > 0) {
+        failed = await runTestem({
+            ...options,
+            suite,
+            processOptions,
+            context: browserContexts
+        }) || failed;
     }
+
+    return failed;
 }
 
 /**
@@ -410,15 +474,14 @@ async function runIntreactiveTestSuite(options) {
         }
         return acc;
     }, { unit: [], visual: [] });
-    _.reduce(tests, async (queue, files, suite) => {
-        await queue;
+    return Promise.all(_.map(tests, (files, suite) => {
         if (files === true) {
             return test(suite, null, options);
         }
         else if (Array.isArray(files) && files.length > 0) {
             return test(suite, files, options);
         }
-    }, Promise.resolve());
+    }));
 }
 
 program
@@ -474,24 +537,28 @@ program
     .option('-p, --port')
     .option('-o, --out <out>', 'path to report test results to')
     .option('--clear-cache', 'clear CLI test cache', false)
-    .action((options) => {
+    .action(async (options) => {
         if (options.clearCache) {
             fs.removeSync(CLI_CACHE);
         }
         if (options.all) {
             options.suite = ['unit', 'visual'];
         }
+        const results = [];
         if (options.suite) {
-            _.reduce(options.suite, async (queue, suite) => {
-                await queue;
+            results.push(...await Promise.all(_.map(options.suite, (suite) => {
                 return test(suite, null, options);
-            }, Promise.resolve());
+            })));
         }
         else if (options.file) {
-            test(options.file.startsWith('visual') ? 'visual' : 'unit', [`test/${options.file}`], options);
+            results.push(await test(options.file.startsWith('visual') ? 'visual' : 'unit', [`test/${options.file}`], options));
         }
         else {
-            runIntreactiveTestSuite(options);
+            results.push(...await runIntreactiveTestSuite(options));
+        }
+        if (_.some(results)) {
+            // inform ci that tests have failed
+            process.exit(1);
         }
     });