Integrate Test262

.travis.yml

@@ -6,22 +6,27 @@ node_js:
   - "4"
 
 env:
-  - JOB=test
+  global:
+    - PATH=$HOME/.yarn/bin:$PATH
+    - JOB=test
 
 before_install:
+  - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 0.28.4
   - yarn global add greenkeeper-lockfile@1
 
 before_script:
   - greenkeeper-lockfile-update
   - 'if [ "$JOB" = "babel-test" ]; then make bootstrap-babel ; fi'
   - 'if [ "$JOB" = "flow-test" ]; then make bootstrap-flow ; fi'
+  - 'if [ "$JOB" = "test262-test" ]; then make bootstrap-test262 ; fi'
   - 'if [ "$JOB" = "test" ]; then yarn run build; fi'
 
 script:
   - 'if [ "$JOB" = "test" ]; then yarn test-only; fi'
   - 'if [ "$JOB" = "lint" ]; then yarn run lint && yarn run flow; fi'
   - 'if [ "$JOB" = "flow-test" ]; then make test-flow; fi'
   - 'if [ "$JOB" = "babel-test" ]; then make test-babel; fi'
+  - 'if [ "$JOB" = "test262-test" ]; then make test-test262; fi'
   - 'if [ "$JOB" = "test-coverage" ]; then yarn run test-coverage; fi'
 
 matrix:
@@ -35,6 +40,8 @@ matrix:
       env: JOB=babel-test
     - node_js: "lts/*"
       env: JOB=flow-test
+    - node_js: node
+      env: JOB=test262-test
   allow_failures:
     - node_js: "lts/*"
       env: JOB=flow-test

.yarnrc

@@ -0,0 +1 @@
+workspaces-experimental true

Makefile

@@ -1,4 +1,5 @@
 MAKEFLAGS = -j1
+TEST262_COMMIT = 4ea2931f169d4a4b5a9a4a7c731cc92bf7b3e13c
 
 export NODE_ENV = test
 
@@ -30,3 +31,14 @@ bootstrap-flow: clean
 
 test-flow:
 	node scripts/run_flow_tests.js
+
+bootstrap-test262: clean
+	mkdir ./build
+	git clone https://github.com/tc39/test262.git ./build/test262
+	cd build/test262 && git checkout $(TEST262_COMMIT)
+
+test-test262:
+	node scripts/run_test262.js
+
+test-test262-update-whitelist:
+	node scripts/run_test262.js --update-whitelist

package.json

@@ -36,6 +36,7 @@
     "eslint-plugin-flowtype": "^2.34.0",
     "eslint-plugin-prettier": "^2.1.2",
     "flow-bin": "^0.52.0",
+    "graceful-fs": "^4.1.11",
     "husky": "^0.14.1",
     "lint-staged": "^4.0.0",
     "nyc": "^11.0.3",
@@ -45,7 +46,8 @@
     "rollup-plugin-babel": "3.0.0-alpha.15",
     "rollup-plugin-node-resolve": "^3.0.0",
     "rollup-watch": "^4.0.0",
-    "unicode-9.0.0": "~0.7.0"
+    "unicode-9.0.0": "~0.7.0",
+    "util.promisify": "^1.0.0"
   },
   "bin": {
     "babylon": "./bin/babylon.js"

scripts/run_test262.js

@@ -0,0 +1,113 @@
+"use strict";
+
+const path = require("path");
+const chalk = require("chalk");
+const utils = require("./run_test262_utils");
+
+const testDir = path.join(__dirname, "..", "build", "test262", "test");
+const whitelistFile = path.join(__dirname, "test262_whitelist.txt");
+const plugins = ["asyncGenerators", "objectRestSpread"];
+const shouldUpdate = process.argv.indexOf("--update-whitelist") > -1;
+
+Promise.all([utils.getTests(testDir), utils.getWhitelist(whitelistFile)])
+  .then(function([tests, whitelist]) {
+    const total = tests.length;
+    const reportInc = Math.floor(total / 20);
+
+    console.log(`Now running ${total} tests...`);
+
+    const results = tests.map(function(test, idx) {
+      if (idx % reportInc === 0) {
+        console.log(`> ${Math.round(100 * idx / total)}% complete`);
+      }
+
+      return utils.runTest(test, plugins);
+    });
+
+    return utils.interpret(results, whitelist);
+  })
+  .then(function(summary) {
+    const goodnews = [
+      summary.allowed.success.length + " valid programs parsed without error",
+      summary.allowed.failure.length +
+        " invalid programs produced a parsing error",
+      summary.allowed.falsePositive.length +
+        " invalid programs did not produce a parsing error" +
+        " (and allowed by the whitelist file)",
+      summary.allowed.falseNegative.length +
+        " valid programs produced a parsing error" +
+        " (and allowed by the whitelist file)",
+    ];
+    const badnews = [];
+    const badnewsDetails = [];
+
+    void [
+      {
+        tests: summary.disallowed.success,
+        label:
+          "valid programs parsed without error" +
+          " (in violation of the whitelist file)",
+      },
+      {
+        tests: summary.disallowed.failure,
+        label:
+          "invalid programs produced a parsing error" +
+          " (in violation of the whitelist file)",
+      },
+      {
+        tests: summary.disallowed.falsePositive,
+        label:
+          "invalid programs did not produce a parsing error" +
+          " (without a corresponding entry in the whitelist file)",
+      },
+      {
+        tests: summary.disallowed.falseNegative,
+        label:
+          "valid programs produced a parsing error" +
+          " (without a corresponding entry in the whitelist file)",
+      },
+      {
+        tests: summary.unrecognized,
+        label: "non-existent programs specified in the whitelist file",
+      },
+    ].forEach(function({ tests, label }) {
+      if (!tests.length) {
+        return;
+      }
+
+      const desc = tests.length + " " + label;
+
+      badnews.push(desc);
+      badnewsDetails.push(desc + ":");
+      badnewsDetails.push(
+        ...tests.map(function(test) {
+          return test.id;
+        })
+      );
+    });
+
+    console.log("Testing complete.");
+    console.log("Summary:");
+    console.log(chalk.green(goodnews.join("\n").replace(/^/gm, " ✔ ")));
+
+    if (!summary.passed) {
+      console.log("");
+      console.log(chalk.red(badnews.join("\n").replace(/^/gm, " ✘ ")));
+      console.log("");
+      console.log("Details:");
+      console.log(badnewsDetails.join("\n").replace(/^/gm, "   "));
+    }
+
+    if (shouldUpdate) {
+      return utils.updateWhitelist(whitelistFile, summary).then(function() {
+        console.log("");
+        console.log("Whitelist file updated.");
+      });
+    } else {
+      process.exitCode = summary.passed ? 0 : 1;
+    }
+  })
+  .catch(function(err) {
+    console.error(err);
+    process.exitCode = 1;
+  });