Skip to content

Commit

Permalink
Refactor TranspilingClosureBundler into a ClosureBundler and an encap…
Browse files Browse the repository at this point in the history 
…sulated Transpiler.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=139382601
  • Loading branch information
@blickly
ejharrington authored and blickly committed Nov 17, 2016
1 parent 44bcaf1 commit 8c1f4d7
Show file tree
Hide file tree
Showing 7 changed files with 409 additions and 194 deletions.
141 changes: 57 additions & 84 deletions src/com/google/javascript/jscomp/deps/ClosureBundler.java
View file
Expand Up @@ -15,47 +15,27 @@
*/ */
package com.google.javascript.jscomp.deps; package com.google.javascript.jscomp.deps;


import com.google.common.base.Strings;
import com.google.common.io.CharSource; import com.google.common.io.CharSource;
import com.google.common.io.Files; import com.google.common.io.Files;
import com.google.javascript.jscomp.transpile.TranspileResult;
import com.google.javascript.jscomp.transpile.Transpiler;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;


/** /**
* A utility class to assist in creating JS bundle files. * A utility class to assist in creating JS bundle files.
*/ */
public class ClosureBundler { public class ClosureBundler {

private boolean useEval = false;
private final Transpiler transpiler;

private EvalMode mode = EvalMode.NORMAL;
private String sourceUrl = null; private String sourceUrl = null;
private String path = "unknown_source"; private String path = "unknown_source";


// TODO(sdh): This cache should be moved out into a higher level, but is
// currently required due to the API that source maps must be accessible
// via just a path (and not the file contents).
private final Map<String, String> sourceMapCache = new ConcurrentHashMap<>();

// TODO(sdh): This causes serious problems with edit-refresh if the bundler
// is used in the wrong scope. The logic must be moved outside this class.
private boolean shouldSendRuntime = true;

public ClosureBundler() { public ClosureBundler() {
this(Transpiler.NULL);
}

public ClosureBundler(Transpiler transpiler) {
this.transpiler = transpiler;
} }


public final ClosureBundler useEval(boolean useEval) { public final ClosureBundler useEval(boolean useEval) {
this.mode = useEval ? EvalMode.EVAL : EvalMode.NORMAL; this.useEval = useEval;
return this; return this;
} }


Expand Down Expand Up @@ -98,19 +78,10 @@ public void appendTo(
Appendable out, Appendable out,
DependencyInfo info, DependencyInfo info,
CharSource content) throws IOException { CharSource content) throws IOException {
// TODO(sdh): Move this logic into the bundle manager.
if (shouldSendRuntime) {
String runtime = transpiler.runtime();
if (!runtime.isEmpty()) {
mode.appendTraditional(runtime, out, null);
}
shouldSendRuntime = false;
}

if (info.isModule()) { if (info.isModule()) {
mode.appendGoogModule(transpile(content.read()), out, sourceUrl); appendGoogModule(out, content);
} else { } else {
mode.appendTraditional(transpile(content.read()), out, sourceUrl); appendTraditional(out, content);
} }
} }


Expand All @@ -119,60 +90,44 @@ public void appendTo(
* method. * method.
*/ */
public String getSourceMap(String path) { public String getSourceMap(String path) {
return Strings.nullToEmpty(sourceMapCache.get(path)); return "";
} }


private String transpile(String s) { private void appendTraditional(Appendable out, CharSource contents)
TranspileResult result = transpiler.transpile(path, s); throws IOException {
sourceMapCache.put(path, result.sourceMap()); if (useEval) {
return result.transpiled(); out.append("(0,eval(\"");
append(out, Mode.ESCAPED, contents);
appendSourceUrl(out, Mode.ESCAPED);
out.append("\"));\n");
} else {
append(out, Mode.NORMAL, contents);
appendSourceUrl(out, Mode.NORMAL);
}
} }


private enum EvalMode { private void appendGoogModule(Appendable out, CharSource contents)
EVAL { throws IOException {
@Override if (useEval) {
void appendTraditional(String s, Appendable out, String sourceUrl) throws IOException { out.append("goog.loadModule(\"");
out.append("(0,eval(\""); append(out, Mode.ESCAPED, contents);
EscapeMode.ESCAPED.append(s, out); appendSourceUrl(out, Mode.ESCAPED);
appendSourceUrl(out, EscapeMode.ESCAPED, sourceUrl); out.append("\");\n");
out.append("\"));\n"); } else {
} // add the prefix on the first line so the line numbers aren't affected.

out.append(
@Override "goog.loadModule(function(exports) {"
void appendGoogModule(String s, Appendable out, String sourceUrl) throws IOException { + "'use strict';");
out.append("goog.loadModule(\""); append(out, Mode.NORMAL, contents);
EscapeMode.ESCAPED.append(s, out); out.append(
appendSourceUrl(out, EscapeMode.ESCAPED, sourceUrl); "\n" // terminate any trailing single line comment.
out.append("\");\n"); + ";" // terminate any trailing expression.
} + "return exports;});\n");
}, appendSourceUrl(out, Mode.NORMAL);
NORMAL { }
@Override
void appendTraditional(String s, Appendable out, String sourceUrl) throws IOException {
EscapeMode.NORMAL.append(s, out);
appendSourceUrl(out, EscapeMode.NORMAL, sourceUrl);
}

@Override
void appendGoogModule(String s, Appendable out, String sourceUrl) throws IOException {
// add the prefix on the first line so the line numbers aren't affected.
out.append(
"goog.loadModule(function(exports) {"
+ "'use strict';");
EscapeMode.NORMAL.append(s, out);
out.append(
"\n" // terminate any trailing single line comment.
+ ";" // terminate any trailing expression.
+ "return exports;});\n");
appendSourceUrl(out, EscapeMode.NORMAL, sourceUrl);
}
};

abstract void appendTraditional(String s, Appendable out, String sourceUrl) throws IOException;
abstract void appendGoogModule(String s, Appendable out, String sourceUrl) throws IOException;
} }


private enum EscapeMode { private enum Mode {
ESCAPED { ESCAPED {
@Override void append(String s, Appendable out) throws IOException { @Override void append(String s, Appendable out) throws IOException {
out.append(SourceCodeEscapers.javascriptEscaper().escape(s)); out.append(SourceCodeEscapers.javascriptEscaper().escape(s));
Expand All @@ -187,8 +142,17 @@ private enum EscapeMode {
abstract void append(String s, Appendable out) throws IOException; abstract void append(String s, Appendable out) throws IOException;
} }


private static void appendSourceUrl(Appendable out, EscapeMode mode, String sourceUrl) private void append(Appendable out, Mode mode, String s) throws IOException {
String transformed = transformInput(s, path);
mode.append(transformed, out);
}

private void append(Appendable out, Mode mode, CharSource cs)
throws IOException { throws IOException {
append(out, mode, cs.read());
}

private void appendSourceUrl(Appendable out, Mode mode) throws IOException {
if (sourceUrl == null) { if (sourceUrl == null) {
return; return;
} }
Expand All @@ -197,4 +161,13 @@ private static void appendSourceUrl(Appendable out, EscapeMode mode, String sour
// but source URLs generally aren't valid JS inputs. // but source URLs generally aren't valid JS inputs.
mode.append(toAppend, out); mode.append(toAppend, out);
} }

/**
* Template method. Subclasses that need to transform the inputs should override this method.
* (For example, {@link TranspilingClosureBundler#transformInput} transpiles inputs from ES6
* to ES5.)
*/
protected String transformInput(String input, String path) {
return input;
}
} }
199 changes: 199 additions & 0 deletions src/com/google/javascript/jscomp/deps/TranspilingClosureBundler.java
View file
@@ -0,0 +1,199 @@
/*
* Copyright 2016 The Closure Compiler Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.javascript.jscomp.deps;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.ImmutableList;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.google.common.io.CharSource;
import com.google.common.util.concurrent.UncheckedExecutionException;
import com.google.javascript.jscomp.Compiler;
import com.google.javascript.jscomp.CompilerOptions;
import com.google.javascript.jscomp.CompilerOptions.LanguageMode;
import com.google.javascript.jscomp.PropertyRenamingPolicy;
import com.google.javascript.jscomp.Result;
import com.google.javascript.jscomp.SourceFile;
import com.google.javascript.jscomp.VariableRenamingPolicy;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import javax.annotation.concurrent.NotThreadSafe;

/**
* {@link ClosureBundler} that transpiles its sources.
*/
@NotThreadSafe
public final class TranspilingClosureBundler extends ClosureBundler {
private static final HashFunction HASH_FUNCTION = Hashing.goodFastHash(64);
private static final int DEFAULT_CACHE_SIZE = 100;
/**
* Cache recent transpilations, keyed by the hash code of the input
* to avoid storing the whole input.
*/
@VisibleForTesting final Cache<Long, String> cachedTranspilations;

// TODO(sdh): Not all transpilation requires the runtime, only inject if actually needed.
private final String es6Runtime;
private boolean needToBundleEs6Runtime = true;
// Whether to inline source map info directly into the output, in a "// #sourceMappingUrl"
// comment. This bloats the size of the transpiled output, but it allows the server to avoid
// serving the source map separately.
private final boolean inlineSourceMap;
// Map of source paths to generated source map paths.
private final Map<String, String> sourceMapCache = new ConcurrentHashMap<>();

public TranspilingClosureBundler() {
this(getEs6Runtime());
}

/**
* Creates a new bundler that transpile the sources from ES6 to ES5.
*
* @param transpilationCache The cache to use to store already transpiled files
*/
public TranspilingClosureBundler(
Cache<Long, String> transpilationCache, boolean inlineSourceMap) {
this(getEs6Runtime(), transpilationCache, inlineSourceMap);
}

@VisibleForTesting
TranspilingClosureBundler(String es6Runtime) {
this(
es6Runtime,
CacheBuilder.newBuilder().maximumSize(DEFAULT_CACHE_SIZE).<Long, String>build(),
true);
}

@VisibleForTesting
TranspilingClosureBundler(
String es6Runtime, Cache<Long, String> transpilationCache, boolean inlineSourceMap) {
this.es6Runtime = es6Runtime;
this.cachedTranspilations = transpilationCache;
this.inlineSourceMap = inlineSourceMap;
}

@Override
public void appendTo(Appendable out, DependencyInfo info, CharSource content) throws IOException {
if (needToBundleEs6Runtime) {
// Piggyback on the first call to transformInput to include the ES6 runtime as well.
super.appendTo(out, SimpleDependencyInfo.EMPTY, CharSource.wrap(es6Runtime));
needToBundleEs6Runtime = false;
}
super.appendTo(out, info, content);
}

private static CompilerOptions getOptions() {
CompilerOptions options = new CompilerOptions();
options.setLanguageIn(LanguageMode.ECMASCRIPT6_STRICT);
options.setLanguageOut(LanguageMode.ECMASCRIPT5);
// Quoting keyword properties is only needed in ES3, so basically only in IE8.
// But we set it explicitly here because the way the test bundler works, it invokes
// the compiler without giving information about the browser, so we have to quote
// every time to be safe :-/
options.setQuoteKeywordProperties(true);
options.setSkipNonTranspilationPasses(true);
options.setVariableRenaming(VariableRenamingPolicy.OFF);
options.setPropertyRenaming(PropertyRenamingPolicy.OFF);
options.setWrapGoogModulesForWhitespaceOnly(false);
options.setPrettyPrint(true);
options.setSourceMapOutputPath("/dev/null");
options.setSourceMapIncludeSourcesContent(true);
return options;
}

@Override
protected String transformInput(final String js, final String path) {
try {
// Don't use built-in hashCode to decrease the likelihood of a collision.
long hashCode = HASH_FUNCTION.hashString(js, StandardCharsets.UTF_8).asLong();
return cachedTranspilations.get(
hashCode,
new Callable<String>() {
@Override
public String call() throws IOException {
// Neither the compiler nor the options is thread safe, so they can't be
// saved as instance state.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Compiler compiler = new Compiler(new PrintStream(baos));
SourceFile sourceFile = SourceFile.fromCode(path, js);
Result result =
compiler.<SourceFile, SourceFile>compile(
ImmutableList.<SourceFile>of(), ImmutableList.of(sourceFile), getOptions());
if (compiler.getErrorManager().getErrorCount() > 0) {
String message;
try {
message = baos.toString(StandardCharsets.UTF_8.name());
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
throw new IllegalStateException(message);
}
if (!result.transpiledFiles.contains(sourceFile)) {
return js;
}
StringBuilder source = new StringBuilder().append(compiler.toSource());
StringBuilder sourceMap = new StringBuilder();
compiler.getSourceMap().appendTo(sourceMap, path);
sourceMapCache.put(path, sourceMap.toString());
if (inlineSourceMap) {
source
.append("\n//# sourceMappingURL=data:,")
.append(URLEncoder.encode(sourceMap.toString(), "UTF-8").replace("+", "%20"));
}
return source.append("\n").toString();
}
});
} catch (ExecutionException | UncheckedExecutionException e) {
// IllegalStateExceptions thrown from the callable above will end up here as
// UncheckedExecutionExceptions, per the contract of Cache#get. Throw the underlying
// IllegalStateException so that the compiler error message is at the top of the stack trace.
if (e.getCause() instanceof IllegalStateException) {
throw (IllegalStateException) e.getCause();
} else {
throw Throwables.propagate(e);
}
}
}

@Override
public String getSourceMap(final String path) {
return sourceMapCache.get(path);
}

/** Generates the runtime by requesting the "es6_runtime" library from the compiler. */
private static String getEs6Runtime() {
CompilerOptions options = getOptions();
options.setLanguageOut(LanguageMode.ECMASCRIPT3); // change .delete to ['delete']
options.setForceLibraryInjection(ImmutableList.of("es6_runtime"));
Compiler compiler = new Compiler();
SourceFile sourceFile = SourceFile.fromCode("source", "");
compiler.<SourceFile, SourceFile>compile(
ImmutableList.<SourceFile>of(), ImmutableList.<SourceFile>of(sourceFile), options);
return compiler.toSource();
}
}

0 comments on commit 8c1f4d7

Please sign in to comment.