Parse //# sourceMappingURL= comments that contain inline source maps.

Standalone source maps can already be specified using --source_map_input. This adds support for inline source maps using data URLs with base64 encoding. Inline source maps make it easier to preserve the source map / file association across build steps and file transfers. Parsing inline source maps can be disabled with the flag `--parse_inline_source_maps=false`. Merge pull request #1982 from mprobst/closure-compiler Closes #1982 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=132101955
google · Sep 2, 2016 · 91f2de5 · 91f2de5
1 parent bc633f4
commit 91f2de5
Show file tree

Hide file tree

Showing 14 changed files with 154 additions and 34 deletions.
diff --git a/src/com/google/JsComp.gwt.xml b/src/com/google/JsComp.gwt.xml
@@ -16,6 +16,7 @@
 <module rename-to="jscomp">
   <inherits name="com.google.common.base.Base"/>
   <inherits name="com.google.common.collect.Collect"/>
+  <inherits name="com.google.common.io.Io"/>
   <inherits name="com.google.gwt.json.JSON"/>
   <source path="debugging/sourcemap"/>
   <source path="javascript/jscomp" excludes=".svn,.git,**/ant/**,**/refactoring/**,**/webservice/**,**/testing/**,**/linker/**"/>

diff --git a/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java b/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java
@@ -43,7 +43,6 @@
 import com.google.javascript.rhino.Node;
 import com.google.javascript.rhino.TokenStream;
 import com.google.protobuf.CodedOutputStream;
-
 import java.io.BufferedInputStream;
 import java.io.BufferedWriter;
 import java.io.Closeable;
@@ -71,7 +70,6 @@
 import java.util.logging.Level;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
-
 import javax.annotation.Nullable;
 
 /**
@@ -382,6 +380,7 @@ protected void setRunOptions(CompilerOptions options) throws IOException {
     options.sourceMapDetailLevel = config.sourceMapDetailLevel;
     options.sourceMapFormat = config.sourceMapFormat;
     options.sourceMapLocationMappings = config.sourceMapLocationMappings;
+    options.parseInlineSourceMaps = config.parseInlineSourceMaps;
     options.applyInputSourceMaps = config.applyInputSourceMaps;
 
     ImmutableMap.Builder<String, SourceMapInput> inputSourceMaps
@@ -2064,6 +2063,14 @@ public CommandLineConfig setSourceMapInputFiles(
       return this;
     }
 
+    private boolean parseInlineSourceMaps = false;
+
+    public CommandLineConfig setParseInlineSourceMaps(
+        boolean parseInlineSourceMaps) {
+      this.parseInlineSourceMaps = parseInlineSourceMaps;
+      return this;
+    }
+
     private String variableMapInputFile = "";
 
     /**

diff --git a/src/com/google/javascript/jscomp/AbstractCompiler.java b/src/com/google/javascript/jscomp/AbstractCompiler.java
@@ -470,6 +470,12 @@ abstract void updateGlobalVarReferences(Map<Var, ReferenceCollection>
    */
   abstract Set<String> getExternProperties();
 
+  /**
+   * Adds a {@link SourceMapInput} for the given {@code sourceFileName}, to be used for error
+   * reporting and source map combining.
+   */
+  public abstract void addInputSourceMap(String name, SourceMapInput sourceMap);
+
   abstract void addComments(String filename, List<Comment> comments);
 
   /**

diff --git a/src/com/google/javascript/jscomp/CommandLineRunner.java b/src/com/google/javascript/jscomp/CommandLineRunner.java
@@ -303,6 +303,12 @@ private static class Flags {
         + "(i.e. input-file-path|input-source-map)")
     private List<String> sourceMapInputs = new ArrayList<>();
 
+    @Option(name = "--parse_inline_source_maps",
+        handler = BooleanOptionHandler.class,
+        hidden = true,
+        usage = "Parse inline source maps (//# sourceMappingURL=data:...)")
+    private Boolean parseInlineSourceMaps = true;
+
     @Option(name = "--apply_input_source_maps",
         handler = BooleanOptionHandler.class,
         hidden = true,
@@ -1280,6 +1286,7 @@ private void initConfigFromFlags(String[] args, PrintStream out, PrintStream err
     List<FlagEntry<JsSourceType>> mixedSources = null;
     List<LocationMapping> mappings = null;
     ImmutableMap<String, String> sourceMapInputs = null;
+    boolean parseInlineSourceMaps = false;
     boolean applyInputSourceMaps = false;
     try {
       flags.parse(processedArgs);
@@ -1293,6 +1300,7 @@ private void initConfigFromFlags(String[] args, PrintStream out, PrintStream err
       mixedSources = flags.getMixedJsSources();
       mappings = flags.getSourceMapLocationMappings();
       sourceMapInputs = flags.getSourceMapInputs();
+      parseInlineSourceMaps = flags.parseInlineSourceMaps;
       applyInputSourceMaps = flags.applyInputSourceMaps;
     } catch (CmdLineException e) {
       reportError(e.getMessage());
@@ -1428,6 +1436,7 @@ private void initConfigFromFlags(String[] args, PrintStream out, PrintStream err
           .setSourceMapFormat(flags.sourceMapFormat)
           .setSourceMapLocationMappings(mappings)
           .setSourceMapInputFiles(sourceMapInputs)
+          .setParseInlineSourceMaps(parseInlineSourceMaps)
           .setApplyInputSourceMaps(applyInputSourceMaps)
           .setWarningGuards(Flags.guardLevels)
           .setDefine(flags.define)

diff --git a/src/com/google/javascript/jscomp/Compiler.java b/src/com/google/javascript/jscomp/Compiler.java
@@ -166,6 +166,10 @@ public SourceFile apply(String filename) {
   private ConcurrentHashMap<String, SourceFile> sourceMapOriginalSources
       = new ConcurrentHashMap<>();
 
+  /** Configured {@link SourceMapInput}s, plus any source maps discovered in source files. */
+  private final ConcurrentHashMap<String, SourceMapInput> inputSourceMaps =
+      new ConcurrentHashMap<>();
+
   // Map from filenames to lists of all the comments in each file.
   private Map<String, List<Comment>> commentsPerFile = new HashMap<>();
 
@@ -483,6 +487,7 @@ public <T extends SourceFile> void initModules(
    * Do any initialization that is dependent on the compiler options.
    */
   private void initBasedOnOptions() {
+    inputSourceMaps.putAll(options.inputSourceMaps);
     // Create the source map if necessary.
     if (options.sourceMapOutputPath != null) {
       sourceMap = options.sourceMapFormat.getInstance();
@@ -1740,6 +1745,7 @@ Node parseSyntheticCode(String fileName, String js) {
   @Override
   Node parseTestCode(String js) {
     initCompilerOptionsIfTesting();
+    initBasedOnOptions();
     CompilerInput input = new CompilerInput(
         SourceFile.fromCode("[testcode]", js));
     if (inputsById == null) {
@@ -2253,7 +2259,8 @@ protected Config createConfig(Config.LanguageMode mode) {
             options.canContinueAfterErrors()
                 ? Config.RunMode.KEEP_GOING
                 : Config.RunMode.STOP_AFTER_ERROR,
-            options.extraAnnotationNames);
+            options.extraAnnotationNames,
+            options.parseInlineSourceMaps);
     return config;
   }
 
@@ -2375,10 +2382,18 @@ SourceFile getSourceFileByName(String sourceName) {
     return null;
   }
 
+  @Override
+  public void addInputSourceMap(String sourceFileName, SourceMapInput inputSourceMap) {
+    inputSourceMaps.put(sourceFileName, inputSourceMap);
+  }
+
   @Override
   public OriginalMapping getSourceMapping(String sourceName, int lineNumber,
       int columnNumber) {
-    SourceMapInput sourceMap = options.inputSourceMaps.get(sourceName);
+    if (sourceName == null) {
+      return null;
+    }
+    SourceMapInput sourceMap = inputSourceMaps.get(sourceName);
     if (sourceMap == null) {
       return null;
     }

diff --git a/src/com/google/javascript/jscomp/CompilerOptions.java b/src/com/google/javascript/jscomp/CompilerOptions.java
@@ -954,6 +954,11 @@ public void setTracerMode(TracerMode mode) {
   public SourceMap.Format sourceMapFormat =
       SourceMap.Format.DEFAULT;
 
+  /**
+   * Whether to parse inline source maps.
+   */
+  boolean parseInlineSourceMaps = true;
+
   /**
    * Whether to apply input source maps to the output, i.e. map back to original inputs from
    * input files that have source maps applied to them.

diff --git a/src/com/google/javascript/jscomp/JsAst.java b/src/com/google/javascript/jscomp/JsAst.java
@@ -24,7 +24,6 @@
 import com.google.javascript.rhino.IR;
 import com.google.javascript.rhino.InputId;
 import com.google.javascript.rhino.Node;
-
 import java.io.IOException;
 import java.util.ArrayList;
 
@@ -153,6 +152,12 @@ private void parse(AbstractCompiler compiler) {
       if (compiler.getOptions().preservesDetailedSourceInfo()) {
         compiler.addComments(sourceFile.getName(), result.comments);
       }
+      if (result.sourceMap != null) {
+        String sourceMapName = sourceFile.getName() + ".inline.map";
+        SourceMapInput sourceMapInput =
+            new SourceMapInput(SourceFile.fromCode(sourceMapName, result.sourceMap));
+        compiler.addInputSourceMap(sourceFile.getName(), sourceMapInput);
+      }
     } catch (IOException e) {
       compiler.report(
           JSError.make(AbstractCompiler.READ_ERROR, sourceFile.getName()));

diff --git a/src/com/google/javascript/jscomp/gwt/client/JsfileParser.java b/src/com/google/javascript/jscomp/gwt/client/JsfileParser.java
@@ -41,16 +41,13 @@
 import com.google.javascript.rhino.ErrorReporter;
 import com.google.javascript.rhino.InputId;
 import com.google.javascript.rhino.Node;
-
-import jsinterop.annotations.JsFunction;
-import jsinterop.annotations.JsMethod;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
-
 import javax.annotation.Nullable;
+import jsinterop.annotations.JsFunction;
+import jsinterop.annotations.JsMethod;
 
 /**
  * GWT module to parse files for dependency and
@@ -217,7 +214,8 @@ private static FileInfo parse(String code, String filename, @Nullable Reporter r
             Config.LanguageMode.ECMASCRIPT6,
             Config.JsDocParsing.INCLUDE_DESCRIPTIONS_NO_WHITESPACE,
             Config.RunMode.KEEP_GOING,
-            /* extraAnnotationNames */ ImmutableSet.<String>of());
+            /* extraAnnotationNames */ ImmutableSet.<String>of(),
+            /* parseInlineSourceMaps */ true);
 
     SourceFile source = SourceFile.fromCode(filename, code);
     FileInfo info = new FileInfo(errorReporter);

diff --git a/src/com/google/javascript/jscomp/parsing/Config.java b/src/com/google/javascript/jscomp/parsing/Config.java
@@ -19,7 +19,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.javascript.jscomp.parsing.parser.FeatureSet;
-
 import java.util.Set;
 
 /**
@@ -114,21 +113,29 @@ public enum RunMode {
    */
   final LanguageMode languageMode;
 
+  /**
+   * Parse inline source maps (//# sourceMappingURL=data:...).
+   */
+  final boolean parseInlineSourceMaps;
+
   Config(Set<String> annotationWhitelist, Set<String> suppressionNames, LanguageMode languageMode) {
     this(
         annotationWhitelist,
         JsDocParsing.TYPES_ONLY,
         RunMode.STOP_AFTER_ERROR,
         suppressionNames,
-        languageMode);
+        languageMode,
+        false);
   }
 
   Config(
       Set<String> annotationWhitelist,
       JsDocParsing parseJsDocDocumentation,
       RunMode keepGoing,
       Set<String> suppressionNames,
-      LanguageMode languageMode) {
+      LanguageMode languageMode,
+      boolean parseInlineSourceMaps) {
+    this.parseInlineSourceMaps = parseInlineSourceMaps;
     this.annotationNames = buildAnnotationNames(annotationWhitelist);
     this.parseJsDocDocumentation = parseJsDocDocumentation;
     this.keepGoing = keepGoing;

diff --git a/src/com/google/javascript/jscomp/parsing/ParserRunner.java b/src/com/google/javascript/jscomp/parsing/ParserRunner.java
@@ -33,11 +33,11 @@
 import com.google.javascript.rhino.Node;
 import com.google.javascript.rhino.SimpleSourceFile;
 import com.google.javascript.rhino.StaticSourceFile;
-
 import java.util.HashSet;
 import java.util.List;
 import java.util.ResourceBundle;
 import java.util.Set;
+import javax.annotation.Nullable;
 
 /** parser runner */
 public final class ParserRunner {
@@ -59,14 +59,16 @@ public static Config createConfig(LanguageMode languageMode,
         languageMode,
         JsDocParsing.TYPES_ONLY,
         RunMode.STOP_AFTER_ERROR,
-        extraAnnotationNames);
+        extraAnnotationNames,
+        true);
   }
 
   public static Config createConfig(
       LanguageMode languageMode,
       JsDocParsing jsdocParsingMode,
       RunMode runMode,
-      Set<String> extraAnnotationNames) {
+      Set<String> extraAnnotationNames,
+      boolean parseInlineSourceMaps) {
 
     initResourceConfig();
     Set<String> effectiveAnnotationNames;
@@ -81,7 +83,8 @@ public static Config createConfig(
         jsdocParsingMode,
         runMode,
         suppressionNames,
-        languageMode);
+        languageMode,
+        parseInlineSourceMaps);
   }
 
   public static Set<String> getReservedVars() {
@@ -133,7 +136,7 @@ public static ParseResult parse(
         comments = p.getComments();
       }
     }
-    return new ParseResult(root, comments, features);
+    return new ParseResult(root, comments, features, p.getInlineSourceMap());
   }
 
   // TODO(sdh): this is less useful if we end up needing the node for library version detection
@@ -212,11 +215,14 @@ public static class ParseResult {
     public final Node ast;
     public final List<Comment> comments;
     public final FeatureSet features;
+    @Nullable
+    public final String sourceMap;
 
-    public ParseResult(Node ast, List<Comment> comments, FeatureSet features) {
+    public ParseResult(Node ast, List<Comment> comments, FeatureSet features, String sourceMap) {
       this.ast = ast;
       this.comments = comments;
       this.features = features;
+      this.sourceMap = sourceMap;
     }
   }
 }