diff --git a/.gitignore b/.gitignore index 37bdebb44..9a53b04f9 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ /test/fixture/main-hook/symlink/symlink.js /test/fixture/main-hook/symlink/symlink.mjs /test/fixture/scenario/ava-nyc-tsc/*.js + +/test/_external/test262/.repo +/test/_external/test262/.test diff --git a/.travis.yml b/.travis.yml index aadc2ec7d..1a0802719 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ os: node_js: - 10 - 8 - - 6.2.0 + - 6 cache: directories: diff --git a/package-lock.json b/package-lock.json index e47e34b67..66ea23369 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14636,6 +14636,16 @@ } } }, + "test262-parser": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/test262-parser/-/test262-parser-2.0.7.tgz", + "integrity": "sha1-cztGv3dZ50fq40tbFNajyNIIKt0=", + "dev": true, + "requires": { + "js-yaml": "^3.2.1", + "through": "^2.3.4" + } + }, "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", diff --git a/package.json b/package.json index 8e9bf007f..8128ea527 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "sleep": "^5.2.3", "sqreen": "^1.22.0", "strip-ansi": "^4.0.0", + "test262-parser": "^2.0.7", "trash": "^4.3.0", "typescript": "^3.0.1", "uglify-es": "^3.3.10", diff --git a/script/test.js b/script/test.js index 08039192d..e72bf3d60 100644 --- a/script/test.js +++ b/script/test.js @@ -6,6 +6,7 @@ const ignorePaths = require("./ignore-paths.js") const path = require("path") const trash = require("./trash.js") const uglify = require("uglify-es").minify +const bootstrapTest262 = require("./test262/bootstrap.js") const argv = require("yargs") .boolean("prod") @@ -114,5 +115,6 @@ Promise cleanRepo(), setupNode() ]) + .then(bootstrapTest262) .then(() => runTests()) .then(() => runTests(true)) diff --git a/script/test262/bootstrap.js b/script/test262/bootstrap.js new file mode 100644 index 000000000..891e1dc4c --- /dev/null +++ b/script/test262/bootstrap.js @@ -0,0 +1,18 @@ +"use strict" + +const clone = require("./clone.js") +const copy = require("./copy.js") + +function bootstrap() { + return clone() + .then(copy) + // TODO remove: + // temporary measure and can be safily removed once + // the `dynamic import` test PR is merged into test262 + // -- BEGIN + // .then(clone.dynamicImport) + // .then(copy.dynamicImport) + // -- END +} + +module.exports = bootstrap diff --git a/script/test262/clone.js b/script/test262/clone.js new file mode 100644 index 000000000..c26cfc02f --- /dev/null +++ b/script/test262/clone.js @@ -0,0 +1,46 @@ +"use strict" + +const execa = require("execa") +const trash = require("trash") +const { resolve } = require( "path") + +const rootPath = resolve(".") +const test262Path = resolve(rootPath, "test/_external/test262") +const test262RepoPath = resolve(test262Path, ".repo") +// const test262RepoGitPath = resolve(test262RepoPath, ".git") + +function run(cwd, file, args) { + return execa(file, args, { + cwd, + reject: false + }) +} + +function clone() { + return trash(test262RepoPath) + .then(() => + run(test262Path, "git", ["--version"]) + ) + .then(() => + run(test262Path, "git", ["clone", "--depth", "1", "https://github.com/tc39/test262.git", ".repo"]) + ) +} + +// TODO remove: +// this is just a temporary measure and can be safily removed once +// the `dynamic import` test PR is merged into test262 +// -- BEGIN +clone.dynamicImport = function () { + return run( + test262RepoPath, "git", ["fetch", "origin", "--depth", "1", "pull/1588/head:dynamic-import"] + ) + .then(() => + run(test262RepoPath, "git", ["checkout", "dynamic-import"]) + ) + // .then(() => + // trash(test262RepoGitPath) + // ) +// -- END +} + +module.exports = clone diff --git a/script/test262/copy.js b/script/test262/copy.js new file mode 100644 index 000000000..6bf8b2a30 --- /dev/null +++ b/script/test262/copy.js @@ -0,0 +1,46 @@ +"use strict" + +const trash = require("trash") +const { resolve } = require("path") +const { copy } = require("fs-extra") + +const rootPath = resolve(".") +const test262RepoPath = resolve(rootPath, "test/_external/test262/.repo") +const testPath = resolve(rootPath, "test/_external/test262/.test") + +const testDirs = [ + "test/language/export", + "test/language/import", + "test/language/module-code", + "harness" +] + +function copyTests() { + return trash(testPath) + .then(() => + Promise.all( + testDirs.map((testDir) => + copy(resolve(test262RepoPath, testDir), resolve(testPath, testDir)) + ) + ) + ) +} + +// TODO remove: +// temporary measure and can be safily removed once +// the `dynamic import` test PR is merged into test262 +// -- BEGIN +const testDirsDynamicImport = [ + "test/language/module-code/dynamic-import" +] + +copyTests.dynamicImport = function () { + return Promise.all( + testDirsDynamicImport.map((testDirDynamicImport) => + copy(resolve(test262RepoPath, testDirDynamicImport), resolve(testPath, testDirDynamicImport)) + ) + ) +} +// -- END + +module.exports = copyTests diff --git a/script/test262/files.js b/script/test262/files.js new file mode 100644 index 000000000..7b809edee --- /dev/null +++ b/script/test262/files.js @@ -0,0 +1,32 @@ +"use strict" + +const globby = require("globby") +const { resolve } = require("path") + +const testPath = resolve(__dirname, "../../test/_external/test262/.test") + +exports.getTestFiles = function getTestFiles() { + return globby( + [ + "test/language/export/**/*.js", + "test/language/import/**/*.js", + "test/language/module-code/**/*.js" + ], + { + absolute: true, + cwd: testPath + } + ) +} + +exports.getHarnessFiles = function getHarnessFiles() { + return globby( + [ + "harness/*.js" + ], + { + absolute: true, + cwd: testPath + } + ) +} diff --git a/test/_external/test262/context.js b/test/_external/test262/context.js new file mode 100644 index 000000000..d14876ed1 --- /dev/null +++ b/test/_external/test262/context.js @@ -0,0 +1,44 @@ +import { createContext, Script } from "vm" +import { basename, resolve } from "path" +import test262Parser from "test262-parser" +import fs from "fs-extra" +import globby from "globby" +import { stdout } from "process" + +// TODO: +const testPath = resolve("../test/_external/test262/.test") + +const harnessFiles = [ + "assert.js", + "sta.js", + "doneprintHandle.js", + "fnGlobalObject.js" +] + +function getHarnessFiles() { + return globby.sync( + [ + "harness/*.js" + ], + { + absolute: true, + cwd: testPath + } + ) +} + +const sandbox = createContext(global) + +export default function harnessContext() { + getHarnessFiles() + .filter((file) => + harnessFiles.includes(basename(file)) + ) + .map((filename) => + fs.readFileSync(filename, "utf-8") + ) + .forEach((rawHarness) => { + const { contents } = test262Parser.parseFile(rawHarness) + new Script(contents).runInContext(sandbox) + }) +} diff --git a/test/_external/test262/index.js b/test/_external/test262/index.js new file mode 100644 index 000000000..6d24dc75c --- /dev/null +++ b/test/_external/test262/index.js @@ -0,0 +1,182 @@ +import assert from "assert" +import execa from "execa" +import globby from "globby" +import { basename, resolve, sep } from "path" +import fs from "fs-extra" +import test262Parser from "test262-parser" + +const { execArgv, execPath, versions } = process + +const testPath = resolve(".") +const test262Path = resolve(testPath, "_external/test262/.test") + +const isHarmony = execArgv.includes("--harmony") +const isChakra = Reflect.has(versions, "chakracore") + +function node(args, env) { + return execa(execPath, args, { + cwd: testPath, + env, + reject: false + }) +} + +const nodeArgs = [] + +if (isHarmony) { + nodeArgs.push("--harmony") +} + +function runMain(filename, env, args) { + return node([...nodeArgs, "-r", "../", filename, ...args], env) +} + +const test262Tests = globby.sync( + [ + "test/language/export/**/*.js", + "test/language/import/**/*.js", + "test/language/module-code/**/*.js", + "!**/*_FIXTURE.js" + ], + { + absolute: true, + cwd: test262Path, + // https://github.com/sindresorhus/globby/issues/38 + transform: (entry) => sep === "\\" ? entry.replace(/\//g, "\\") : entry + } +) + +const skiplist = new Map() + +function parseTest(filepath) { + const rawTest = fs.readFileSync(filepath, "utf-8") + + const { + attrs: { + description, + negative, + flags + } } = test262Parser.parseFile(rawTest) + + return { + description, + isAsync: flags && flags.async, + errorType: negative && negative.type + } +} + +function loadSkiplist() { + const content = fs.readFileSync(resolve(test262Path, "../skiplist"), "utf-8") + + content + .split("\n") + .map((line) => line.trim()) + .filter((line) => line !== "") + .reduce((comment, line) => { + if (line.startsWith("#")) { + if (comment) { + throw new Error( + `The skiplist contains multiple comments in consecutive rows: "${comment}" and "${line}". This is not allowed.` + ) + } + + return line + } + + if (! comment) { + throw new Error( + `A reason for skipping is required! None was given for: "${line}".` + ) + } + + // @+harmony + const flag = comment.match(/@[+-][a-z]*/g) + + const skiplistFlag = flag ? flag[0].replace("@", "") : "" + + const fullPath = resolve(test262Path, line) + + if (isSkiplisted(fullPath)) { + throw new Error( + `Same entry in skiplist already exists for: "${line}".` + ) + } + + comment = comment.slice(1).trimLeft() + + skiplist.set(fullPath, { + comment, + skiplistFlag + }) + + return null + }, null) +} + +loadSkiplist() + +function isSkiplisted(test) { + // return skiplist.has(test) + const item = skiplist.get(test) + + if (! item) { + return false + } + + if (isHarmony && item.skiplistFlag === "+harmony") { + return false + } + + if (! isChakra && item.skiplistFlag === "-chakra") { + return false + } + + return true +} + +function skiplistReason(test) { + return skiplist.get(test).comment +} + +const test262Error = "Test262: This statement should not be evaluated." + +describe.only("test262 module tests", function () { + this.timeout(0) + + test262Tests.forEach(function (test262TestPath, index) { + const { description } = parseTest(test262TestPath) + + const skip = isSkiplisted(test262TestPath) + + const skipReason = skip ? `| ${skiplistReason(test262TestPath)}` : "" + + const testfunc = skip ? it.skip : it + + const filename = basename(test262TestPath) + + const ESM_OPTIONS = JSON.stringify({ cjs: false, mode: "all" }) + + testfunc(`[${index}] ${description} (${filename}) ${skipReason}`, function () { + + const { isAsync, errorType } = parseTest(test262TestPath) + + return runMain(resolve(test262Path, "../wrapper.js"), { ESM_OPTIONS }, [test262TestPath, isAsync]) + .then((out) => { + const { stdout, stderr } = out + + if (stdout) { + const { name } = JSON.parse(stdout) + + // possible known "supported" test262 constructors: + // SyntaxError, Test262Error, ReferenceError, TypeError + assert.strictEqual(errorType, name) + } + + if (stderr) { + console.log("stderr ==>", stderr) + assert.fail("possible test262 Error") + } + }) + }) + }) +}) diff --git a/test/_external/test262/skiplist b/test/_external/test262/skiplist new file mode 100644 index 000000000..fa98702af --- /dev/null +++ b/test/_external/test262/skiplist @@ -0,0 +1,180 @@ +# NOT YET SUPPORTED 🚧 (export * as ns from "mod") +test/language/module-code/eval-rqstd-once.js + +# NOT YET SUPPORTED 🚧 (export * as ns from "mod") +test/language/module-code/eval-rqstd-order.js + +# NOT YET SUPPORTED 🚧 (export * as ns from "mod") +test/language/module-code/eval-self-once.js + +# NOT YET SUPPORTED 🚧 (export * as ns from "mod") +test/language/module-code/instn-once.js + +# NOT YET SUPPORTED 🚧 (export * as ns from "mod") +test/language/module-code/instn-star-as-props-dflt-skip.js + +# NOT YET SUPPORTED 🚧 (export * as ns from "mod") +test/language/module-code/instn-star-props-nrml.js + +# NOT YET SUPPORTED 🚧 (export * as ns from "mod") +test/language/module-code/namespace/internals/get-nested-namespace-dflt-skip.js + +# NOT YET SUPPORTED 🚧 (export * as ns from "mod") +test/language/module-code/namespace/internals/get-nested-namespace-props-nrml.js + + + +# CONFIRMED BUG 🐛 (instantiation phase) +test/language/module-code/instn-iee-bndng-fun.js + +# CONFIRMED BUG 🐛 (instantiation phase) +test/language/module-code/instn-iee-bndng-gen.js + +# CONFIRMED BUG 🐛 (instantiation phase) +test/language/module-code/instn-iee-bndng-var.js + +# CONFIRMED BUG 🐛 (instantiation phase) +test/language/module-code/instn-iee-star-cycle.js + +# CONFIRMED BUG 🐛 (instantiation phase) +test/language/module-code/instn-iee-trlng-comma.js + +# CONFIRMED BUG 🐛 (instantiation phase) +test/language/module-code/instn-named-bndng-dflt-named.js + +# CONFIRMED BUG 🐛 (instantiation phase) +test/language/module-code/instn-named-bndng-trlng-comma.js + +# CONFIRMED BUG 🐛 (instantiation phase) +test/language/module-code/instn-named-bndng-var.js + + + +# CONFIRMED BUG 🐛 (cycle import) +test/language/module-code/instn-iee-iee-cycle.js + +# NEEDS INVESTIGATION 🐛 (cycle import) +test/language/module-code/instn-named-iee-cycle.js + +# NEEDS INVESTIGATION 🐛 (cycle import) +test/language/module-code/instn-star-iee-cycle.js + + + +# PRIVATE CLASS FIELD 🔥 @+harmony +test/language/module-code/privatename-not-valid-earlyerr-module-1.js + +# PRIVATE CLASS FIELD 🔥 @+harmony +test/language/module-code/privatename-not-valid-earlyerr-module-2.js + +# PRIVATE CLASS FIELD 🔥 @+harmony +test/language/module-code/privatename-not-valid-earlyerr-module-3.js + +# PRIVATE CLASS FIELD 🔥 @+harmony +test/language/module-code/privatename-not-valid-earlyerr-module-4.js + +# PRIVATE CLASS FIELD 🔥 @+harmony +test/language/module-code/privatename-not-valid-earlyerr-module-5.js + +# PRIVATE CLASS FIELD 🔥 @+harmony +test/language/module-code/privatename-not-valid-earlyerr-module-6.js + +# PRIVATE CLASS FIELD 🔥 @+harmony +test/language/module-code/privatename-not-valid-earlyerr-module-7.js + +# PRIVATE CLASS FIELD 🔥 @+harmony +test/language/module-code/privatename-not-valid-earlyerr-module-8.js + +# PRIVATE CLASS FIELD 🔥 @+harmony +test/language/module-code/privatename-valid-no-earlyerr.js + + + +# NEEDS INVESTIGATION 👀 +test/language/module-code/namespace/internals/delete-exported-init.js + +# NEEDS INVESTIGATION 👀 +test/language/module-code/namespace/internals/get-own-property-str-found-init.js + +# NEEDS INVESTIGATION 👀 +test/language/module-code/namespace/internals/get-str-found-init.js + +# NEEDS INVESTIGATION 👀 +test/language/module-code/namespace/internals/get-str-initialize.js + +# NEEDS INVESTIGATION 👀 +test/language/module-code/namespace/internals/get-str-update.js + +# NEEDS INVESTIGATION 👀 -> only fails on node v6.14.3 +test/language/module-code/namespace/internals/object-keys-binding-uninit.js + + + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-iee-bndng-cls.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-iee-bndng-const.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-iee-bndng-let.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-named-bndng-cls.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-named-bndng-const.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-named-bndng-dflt-cls.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-named-bndng-dflt-expr.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-named-bndng-dflt-star.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-named-bndng-fun.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-named-bndng-gen.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-named-bndng-let.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-star-binding.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/instn-uniq-env-rec.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/define-own-property.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/delete-exported-uninit.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/delete-non-exported.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/enumerate-binding-uninit.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/get-own-property-str-found-uninit.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/get-str-found-uninit.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/object-hasOwnProperty-binding-uninit.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/object-propertyIsEnumerable-binding-uninit.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/set.js + +# CHAKRA ONLY ⚠️ - NEEDS INVESTIGATION 👀 @-chakra +test/language/module-code/namespace/internals/set-prototype-of-null.js diff --git a/test/_external/test262/wrapper.js b/test/_external/test262/wrapper.js new file mode 100644 index 000000000..975de6560 --- /dev/null +++ b/test/_external/test262/wrapper.js @@ -0,0 +1,61 @@ +import { argv, stderr, stdout } from "process" +import harnessContext from "./context.js" + +const [,,testUrl, isAsync] = argv +const _isAsync = isAsync === "true" + +// attach test262 harness onto global scope +harnessContext() + +let _msg + +// global print function expected by test262 +global.print = function print(val) { + _msg = val +} + +const MAX_WAIT = 1000 + +function waitForAsyncTest() { + let wait = 0 + + return new Promise(function time(res, rej) { + if (_msg) { + return res() + } else if (wait === MAX_WAIT) { + return rej("test262: async test timed out.") + } + + setTimeout(() => time(res, rej), wait += 200) + }) +} + +import(testUrl) + .then(() => { + if (! _isAsync) { + return + } + + return waitForAsyncTest() + .then(() => { + if (_msg !== "Test262:AsyncTestComplete") { + throw _msg + } + }) + }) + .catch((e) => { + const name = typeof e === "string" ? e : e.constructor.name + + const send = JSON.stringify({ + name, + message: typeof e !== "string" && e.message, + argv + }) + + stdout.write(send) + }) + .catch((e) => + stderr.write( + `[last resort catch] something happened in the previous catch block: ${e.message}}` + ) + ) diff --git a/test/tests.js b/test/tests.js index 9ed85e8ed..4a02041ce 100644 --- a/test/tests.js +++ b/test/tests.js @@ -19,6 +19,7 @@ import "./main-hook-tests.mjs" import "./require-hook-tests.js" import "./repl-hook-tests.mjs" import "./scenario-tests.mjs" +import "./_external/test262/index.js" const extensions = Object.assign({}, require.extensions)