Add -C/--compile option

README.md

@@ -30,6 +30,8 @@ These options may be passed on the command line or passed to useConfig in your c
 |------------|---------------|
 | -r, --runner | Selects a runner to use. Currently available are `node`, `node-ip`, `jsshell`, and `console`. Config files may also pass a runner constructor whose instances implement the runner API described below.
 | -c, --config | Load a config.js file
+| -C, --compile | Save compiled tests so they can be run directly in the host without the harness. Can specify either "all" or "failures". Default is "all".
+| -o, --output | Output directory for compiled test collateral
 | -e, --consoleCommand | For console runner, sets the command to invoke. Must be in PATH.
 | -p, --consolePrintCommand | For console runner, sets the command to write to console. Used for reporting errors to the harness.
 | -t, --threads | Run this many tests in parallel.
@@ -65,6 +67,9 @@ Different runners may execute tests in different ways. The two basic methods are
 | console | `console` | Runs tests out-of-proc in a generic console host. Works with Node, JSShell, and probably others. You will have to provide -p for normal mode runs (defaults to console.log). Can enable batch mode behavior by providing the createEnv and runBatched configuration options.
 | jsshell | `jsshell` | Subclass of the console runner with defaults for jsshell
 
+## Debugging Test Failures
+Because test262-harness makes significant modifications to a test before ultimately running it, even stack traces are little help when debugging test262 failures. In these cases, the --compile flag is your friend. By passing "--compile failures -o output" to the harness, fully compiled failing tests will be written to disk under the output directory. You can run these files directly in the host you are testing.
+
 ## API
 
 ### Test262

bin/run.js

@@ -11,7 +11,9 @@ var args = require('minimist')(process.argv.slice(2), {
         reporter: 'R',
         threads: 't',
         batch: 'b',
-        config: 'c'
+        config: 'c',
+        compile: 'C',
+        outputDir: 'o'
     }
 });
 
@@ -38,6 +40,10 @@ var Runner = t262.loadRunner();
 if(t262.config.batch === true) t262.config.batch = DEFAULT_BATCH_SIZE;
 if(!t262.config.threads) t262.config.threads = 4;
 if(!t262.config.reporter) t262.config.reporter = 'simple';
+if(t262.config.compile === true) t262.config.compile = 'all';
+if(!t262.config.outputDir && t262.config.compile)
+    throw 'need output directory for compiled collateral';
+if(t262.config.compile && !fs.existsSync(t262.config.outputDir)) fs.mkdirSync(t262.config.outputDir);
 
 var start = Date.now();
 
@@ -57,6 +63,35 @@ var results = _(function(push) {
 }).merge();
 
 
+if(t262.config.compile) {
+    optionsCopy = shallowCopy(t262.config);
+    optionsCopy.batch = false;
+    optionsCopy.compileOnly = true;
+    var compiler = new Runner(optionsCopy);
+
+    _(results).observe().each(function(test) {
+        var compile = t262.config.compile === 'failures' && !test.pass;
+        compile = compile || t262.config.compile === 'all';
+        compile = compile || t262.config.compile === 'passes' && !!test.pass
+
+        if(compile) {
+            var testCopy = shallowCopy(test);
+            compiler.compile(testCopy)
+            var startPath = path.join(process.cwd(), t262.config.outputDir, path.basename(test.file));
+            var currentPath = startPath;
+            var counter = 0;
+            while(fs.existsSync(currentPath)) {
+                currentPath = startPath.replace(/.js$/, "-" + counter++ + ".js");
+            }
+            fs.writeFile(currentPath, testCopy.contents.replace(/\r\n/g, '\n'));
+        }
+    });
+
+    results.on('end', function() {
+        compiler.end();
+    });
+}
+
 if(t262.config.reporter === 'json') {
     results.pipe(jss).pipe(process.stdout);
 } else if(t262.config.reporter === 'tap') {
@@ -122,3 +157,9 @@ function scenarioStream(test) {
         push(null, _.nil);
     })
 }
+
+function shallowCopy(obj) {
+    return Object.keys(obj).reduce(function(v, k) {
+        return v[k] = obj[k], v;
+    }, {});
+}

lib/runners/node.js

@@ -11,31 +11,33 @@ function NodeRunner(args) {
 
     ConsoleRunner.apply(this, arguments);
 
-    this.deps = []; // all env deps provided by nodehost.js
-
-    // HACK: Probably doesn't handle quoted arguments and other
-    // complexities.
-    var parts = args.consoleCommand.split(" ");
-    parts.push(__dirname + '/nodehost.js');
-
-    this._instance = cp.spawn(parts[0], parts.slice(1));
-
-
-    var results = { log: [] };
-
-    _(this._instance.stdout).flatMap(function(data) {
-        return _(data.toString().split(/\r?\n/g));
-    }).each(function(line) {
-        results.log.push(line);
-        switch(line) {
-            case 'test262/done':
-                runner.validateResult(runner._test, results);
-                results = { log: [] };
-
-                runner._testDone();
-                break;
-        }
-    })
+    if(!args.compileOnly) {
+        this.deps = []; // all env deps provided by nodehost.js
+
+        // HACK: Probably doesn't handle quoted arguments and other
+        // complexities.
+        var parts = args.consoleCommand.split(" ");
+        parts.push(__dirname + '/nodehost.js');
+
+        this._instance = cp.spawn(parts[0], parts.slice(1));
+
+
+        var results = { log: [] };
+
+        _(this._instance.stdout).flatMap(function(data) {
+            return _(data.toString().split(/\r?\n/g));
+        }).each(function(line) {
+            results.log.push(line);
+            switch(line) {
+                case 'test262/done':
+                    runner.validateResult(runner._test, results);
+                    results = { log: [] };
+
+                    runner._testDone();
+                    break;
+            }
+        })
+    }
 }
 NodeRunner.prototype = Object.create(ConsoleRunner.prototype);
 NodeRunner.prototype.execute = function(test, cb) {
@@ -45,5 +47,5 @@ NodeRunner.prototype.execute = function(test, cb) {
 }
 
 NodeRunner.prototype.end = function() {
-    this._instance.stdin.end();
+    if(this._instance) this._instance.stdin.end();
 }

package.json

@@ -30,6 +30,7 @@
     "test"
   ],
   "devDependencies": {
+    "rimraf": "^2.2.8",
     "tape": "^3.0.3"
   }
 }

test/test.js

@@ -1,16 +1,39 @@
 var test = require('tape');
 var utils = require('./utils');
 var expected = require('./expected');
+var rimraf = require('rimraf');
+var fs = require('fs');
+var path = require('path');
 
 var runners = [
     '-r node',
     '-r node-ip',
     '-r console -e node'
 ]
+
+// clean testOutput dir
+rimraf.sync('./testOutput');
+
 // default arguments and strict mode
 runners.forEach(function(runner) {
     utils.testResultsCorrect(runner, expected.noTestStrict);
     utils.testResultsCorrect(runner + ' --testStrict', expected);
+    test("compiling failures: " + runner + " -C failures -o testOutput", function(t) {
+        t.plan(1);
+        utils.run(runner + " -C failures -o testOutput", function(res, stderr) {
+            t.ok(fs.existsSync('./testOutput/error.js'), 'testOutput/error.js exists');
+            rimraf.sync('./testOutput');
+        });
+    });
+
+    test("compiling failures: " + runner + " -C failures -o testOutput --testStrict", function(t) {
+        t.plan(2);
+        utils.run(runner + " -C failures -o testOutput --testStrict", function(res, stderr) {
+            t.ok(fs.existsSync('./testOutput/error.js'), 'testOutput/error.js exists');
+            t.ok(fs.existsSync('./testOutput/error-0.js'), 'testOutput/error-0.js exists');
+            rimraf.sync('./testOutput');
+        });
+    });
 });
 
 // batch mode supported by console runner
@@ -30,3 +53,37 @@ utils.testResultsCorrect('-r console -e node -b 5 --testStrict -c test/nodeConfi
     })
 });
 
+test('compile flag defaults to compiling all collateral', function(t) {
+    utils.run("-r node -C -o testOutput", function(res, stderr) {
+        expected.noTestStrict.forEach(function(exp) {
+            var expectedLoc = './testOutput/' + path.basename(exp.file);
+            t.ok(fs.existsSync(expectedLoc), expectedLoc + ' exists');
+        })
+
+        rimraf.sync('./testOutput');
+        t.end();
+    });
+});
+
+test('compile flag defaults to compiling all collateral', function(t) {
+    utils.run("-r node -C all -o testOutput", function(res, stderr) {
+        expected.noTestStrict.forEach(function(exp) {
+            var expectedLoc = './testOutput/' + path.basename(exp.file);
+            t.ok(fs.existsSync(expectedLoc), expectedLoc + ' exists');
+        })
+
+        rimraf.sync('./testOutput');
+        t.end();
+    });
+});
+
+test('compile flag throws without output directory', function(t) {
+    t.plan(2);
+
+    utils.run('-r node -C failures', function(res, stderr) {
+        t.equal(res.length, 0, 'no results');
+        t.ok(stderr.length > 0, 'stderr is present');
+        rimraf.sync('./testOutput');
+    });
+});
+