From fa7cb573c14912e1cb68dbbb5502330c3eeac0cf Mon Sep 17 00:00:00 2001 From: johnplaisted Date: Mon, 29 Jan 2018 14:18:29 -0800 Subject: [PATCH] Use the new ES6 modules to a CJS-like module rewriter in closure bundler to safely concatenate ES6 modules in a bundle. Also use it in the gwt transpiler so the Closure Library can transpile ES6 modules. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=183726148 --- .../jscomp/TranspilationPasses.java | 18 +++++++ .../jscomp/deps/ClosureBundler.java | 22 ++++++-- .../jscomp/transpile/BaseTranspiler.java | 52 +++++++++++++++++++ .../jscomp/deps/ClosureBundlerTest.java | 37 ++++++++++++- 4 files changed, 124 insertions(+), 5 deletions(-) diff --git a/src/com/google/javascript/jscomp/TranspilationPasses.java b/src/com/google/javascript/jscomp/TranspilationPasses.java index 6358b41b165..e04cfeafef4 100644 --- a/src/com/google/javascript/jscomp/TranspilationPasses.java +++ b/src/com/google/javascript/jscomp/TranspilationPasses.java @@ -36,6 +36,10 @@ public static void addEs6ModulePass(List passes) { passes.add(es6RewriteModule); } + public static void addEs6ModuleToCjsPass(List passes) { + passes.add(es6RewriteModuleToCjs); + } + public static void addEs2018Passes(List passes) { passes.add(rewriteObjRestSpread); } @@ -121,6 +125,20 @@ protected FeatureSet featureSet() { } }; + /** Rewrites ES6 modules */ + private static final PassFactory es6RewriteModuleToCjs = + new PassFactory("es6RewriteModuleToCjs", true) { + @Override + protected CompilerPass create(AbstractCompiler compiler) { + return new Es6RewriteModulesToCommonJsModules(compiler); + } + + @Override + protected FeatureSet featureSet() { + return ES8_MODULES; + } + }; + private static final PassFactory rewriteAsyncFunctions = new HotSwapPassFactory("rewriteAsyncFunctions") { @Override diff --git a/src/com/google/javascript/jscomp/deps/ClosureBundler.java b/src/com/google/javascript/jscomp/deps/ClosureBundler.java index c6c6645beb2..175f4bc8a0e 100644 --- a/src/com/google/javascript/jscomp/deps/ClosureBundler.java +++ b/src/com/google/javascript/jscomp/deps/ClosureBundler.java @@ -18,6 +18,7 @@ import com.google.common.base.Strings; import com.google.common.io.CharSource; import com.google.common.io.Files; +import com.google.javascript.jscomp.transpile.BaseTranspiler; import com.google.javascript.jscomp.transpile.TranspileResult; import com.google.javascript.jscomp.transpile.Transpiler; import java.io.File; @@ -33,6 +34,7 @@ public final class ClosureBundler { private final Transpiler transpiler; + private final Transpiler es6ModuleTranspiler; private final EvalMode mode; private final String sourceUrl; @@ -48,8 +50,7 @@ public ClosureBundler() { } public ClosureBundler(Transpiler transpiler) { - this(transpiler, EvalMode.NORMAL, null, "unknown_source", - new ConcurrentHashMap()); + this(transpiler, EvalMode.NORMAL, null, "unknown_source", new ConcurrentHashMap<>()); } private ClosureBundler(Transpiler transpiler, EvalMode mode, String sourceUrl, String path, @@ -59,6 +60,7 @@ private ClosureBundler(Transpiler transpiler, EvalMode mode, String sourceUrl, S this.sourceUrl = sourceUrl; this.path = path; this.sourceMapCache = sourceMapCache; + this.es6ModuleTranspiler = BaseTranspiler.ES_MODULE_TO_CJS_TRANSPILER; } public final ClosureBundler useEval(boolean useEval) { @@ -105,6 +107,8 @@ public void appendTo( CharSource content) throws IOException { if (info.isModule()) { mode.appendGoogModule(transpile(content.read()), out, sourceUrl); + } else if ("es6".equals(info.getLoadFlags().get("module"))) { + mode.appendTraditional(transpileEs6Module(content.read()), out, sourceUrl); } else { mode.appendTraditional(transpile(content.read()), out, sourceUrl); } @@ -115,6 +119,7 @@ public void appendRuntimeTo(Appendable out) throws IOException { if (!runtime.isEmpty()) { mode.appendTraditional(runtime, out, null); } + mode.appendTraditional(es6ModuleTranspiler.runtime(), out, null); } /** @@ -125,12 +130,20 @@ public String getSourceMap(String path) { return Strings.nullToEmpty(sourceMapCache.get(path)); } - private String transpile(String s) { - TranspileResult result = transpiler.transpile(Paths.get(path), s); + private String transpile(String s, Transpiler t) { + TranspileResult result = t.transpile(Paths.get(path), s); sourceMapCache.put(path, result.sourceMap()); return result.transpiled(); } + private String transpile(String s) { + return transpile(s, transpiler); + } + + private String transpileEs6Module(String s) { + return transpile(transpile(s, es6ModuleTranspiler)); + } + private enum EvalMode { EVAL { @Override @@ -172,6 +185,7 @@ void appendGoogModule(String s, Appendable out, String sourceUrl) throws IOExcep }; abstract void appendTraditional(String s, Appendable out, String sourceUrl) throws IOException; + abstract void appendGoogModule(String s, Appendable out, String sourceUrl) throws IOException; } diff --git a/src/com/google/javascript/jscomp/transpile/BaseTranspiler.java b/src/com/google/javascript/jscomp/transpile/BaseTranspiler.java index 315f262bd22..ca64f82e50d 100644 --- a/src/com/google/javascript/jscomp/transpile/BaseTranspiler.java +++ b/src/com/google/javascript/jscomp/transpile/BaseTranspiler.java @@ -25,11 +25,13 @@ import com.google.javascript.jscomp.CompilerOptions.LanguageMode; import com.google.javascript.jscomp.DiagnosticGroup; import com.google.javascript.jscomp.DiagnosticType; +import com.google.javascript.jscomp.Es6RewriteModulesToCommonJsModules; import com.google.javascript.jscomp.PropertyRenamingPolicy; import com.google.javascript.jscomp.Result; import com.google.javascript.jscomp.SourceFile; import com.google.javascript.jscomp.VariableRenamingPolicy; import com.google.javascript.jscomp.bundle.TranspilationException; +import com.google.javascript.rhino.Node; import java.io.IOException; import java.nio.file.Path; @@ -63,6 +65,9 @@ public String runtime() { public static final BaseTranspiler ES5_TRANSPILER = new BaseTranspiler( new CompilerSupplier(), "es6_runtime"); + public static final BaseTranspiler ES_MODULE_TO_CJS_TRANSPILER = + new BaseTranspiler(new EsmToCjsCompilerSupplier(), "modules"); + /** * Wraps the Compiler into a more relevant interface, making it * easy to test the Transpiler without depending on implementation @@ -143,6 +148,53 @@ protected void setOptions(CompilerOptions options) { DiagnosticType.error("JSC_CANNOT_CONVERT", "")); } + /** + * CompilerSupplier that only transforms EcmaScript Modules into a form that can be saftely + * transformed on a file by file basis and concatenated. + */ + public static class EsmToCjsCompilerSupplier extends CompilerSupplier { + @Override + public CompileResult compile(Path path, String code) { + CompilerOptions options = new CompilerOptions(); + options.setSourceMapOutputPath("/dev/null"); + options.setSourceMapIncludeSourcesContent(true); + options.setPrettyPrint(true); + + // Create a compiler and run specifically this one pass on it. + Compiler compiler = compiler(); + compiler.init( + ImmutableList.of(), + ImmutableList.of(SourceFile.fromCode(path.toString(), code)), + options); + compiler.parseForCompilation(); + + boolean transpiled = false; + + if (!compiler.hasErrors() + && compiler.getRoot().getSecondChild().getFirstFirstChild().isModuleBody()) { + new Es6RewriteModulesToCommonJsModules(compiler) + .process(null, compiler.getRoot().getSecondChild()); + compiler.getRoot().getSecondChild().getFirstChild().putBooleanProp(Node.TRANSPILED, true); + transpiled = true; + } + + Result result = compiler.getResult(); + String source = compiler.toSource(); + StringBuilder sourceMap = new StringBuilder(); + if (result.sourceMap != null) { + try { + result.sourceMap.appendTo(sourceMap, path.toString()); + } catch (IOException e) { + // impossible, and not a big deal even if it did happen. + } + } + if (result.errors.length > 0) { + throw new TranspilationException(compiler, result.errors, result.warnings); + } + return new CompileResult(source, transpiled, transpiled ? sourceMap.toString() : ""); + } + } + /** * The source together with the additional compilation results. */ diff --git a/test/com/google/javascript/jscomp/deps/ClosureBundlerTest.java b/test/com/google/javascript/jscomp/deps/ClosureBundlerTest.java index 55c78ee38ef..8da398ca754 100644 --- a/test/com/google/javascript/jscomp/deps/ClosureBundlerTest.java +++ b/test/com/google/javascript/jscomp/deps/ClosureBundlerTest.java @@ -19,6 +19,7 @@ import static org.mockito.Mockito.RETURNS_SMART_NULLS; import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableMap; import com.google.javascript.jscomp.transpile.TranspileResult; import com.google.javascript.jscomp.transpile.Transpiler; import java.io.IOException; @@ -120,8 +121,10 @@ public void testTranspilation() throws IOException { StringBuilder sb = new StringBuilder(); bundler.appendRuntimeTo(sb); bundler.appendTo(sb, MODULE, input); + assertThat(sb.toString()).startsWith("RUNTIME;"); + // Call endsWith because the ES6 module runtime is also injected. assertThat(sb.toString()) - .isEqualTo("RUNTIME;goog.loadModule(function(exports) {'use strict';TRANSPILED;\n" + .endsWith("goog.loadModule(function(exports) {'use strict';TRANSPILED;\n" + ";return exports;});\n"); // Without calling appendRuntimeTo(), the runtime is not included anymore. @@ -131,4 +134,36 @@ public void testTranspilation() throws IOException { .isEqualTo("goog.loadModule(function(exports) {'use strict';TRANSPILED;\n" + ";return exports;});\n"); } + + public void testEs6Module() throws IOException { + String input = + "import {x} from './other.js';\n" + + "export {x as y};" + + "var local;\n" + + "export function foo() { return local; }\n"; + ClosureBundler bundler = new ClosureBundler().withPath("foo.js"); + StringBuilder sb = new StringBuilder(); + bundler.appendRuntimeTo(sb); + bundler.appendTo( + sb, + SimpleDependencyInfo.builder("", "").setLoadFlags(ImmutableMap.of("module", "es6")).build(), + input); + String result = sb.toString(); + // ES6 module runtime should be injected. + assertThat(result).contains("$jscomp.require = createRequire();"); + assertThat(sb.toString()) + .endsWith( + "$jscomp.registerAndLoadModule(function($$require, $$exports, $$module) {\n" + + " Object.defineProperties($$exports, {foo:{enumerable:true, get:function() {\n" + + " return foo;\n" + + " }}, y:{enumerable:true, get:function() {\n" + + " return module$other.x;\n" + + " }}});\n" + + " var module$other = $$require(\"./other.js\");\n" + + " var local;\n" + + " function foo() {\n" + + " return local;\n" + + " }\n" + + "}, \"foo.js\", [\"./other.js\"]);\n"); + } }