From daa73e1fc567b47997aa9565e5596e7a428e136a Mon Sep 17 00:00:00 2001 From: martinprobst Date: Mon, 29 Aug 2016 16:53:39 -0700 Subject: [PATCH] Allow applying input source maps to the generated source map. This allows users to easily map from generated source that's an input to the compiler back into their original source files. E.g. when compiling TypeScript, users would have: input.ts -(a)-> input.js -(b)-> compiled.js Where (a) is the TypeScript compilation operation, and (b) is Closure Compiler. With this option set, the source map generated in (a) is applied to the operation (b), with the final source map mapping back into `input.ts` from locations in `compiled.js`. This also introduces an interface `SourceFileMapping` to avoid increasing the coupling and responsibility of the Compiler object. Merge pull request #1971 from mprobst/closure-compiler Closes https://github.com/google/closure-compiler/pull/1971 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=131651810 --- .../jscomp/AbstractCommandLineRunner.java | 12 +++ .../javascript/jscomp/CommandLineRunner.java | 94 ++++++++++--------- .../google/javascript/jscomp/Compiler.java | 5 +- .../javascript/jscomp/CompilerOptions.java | 10 ++ .../javascript/jscomp/SourceFileMapping.java | 36 +++++++ .../google/javascript/jscomp/SourceMap.java | 26 ++++- .../javascript/jscomp/ant/CompileTask.java | 3 + .../javascript/jscomp/CompilerTest.java | 32 +++++++ 8 files changed, 174 insertions(+), 44 deletions(-) create mode 100644 src/com/google/javascript/jscomp/SourceFileMapping.java diff --git a/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java b/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java index 296c7efe1d3..c81425a4a27 100644 --- a/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java +++ b/src/com/google/javascript/jscomp/AbstractCommandLineRunner.java @@ -382,6 +382,7 @@ protected void setRunOptions(CompilerOptions options) throws IOException { options.sourceMapDetailLevel = config.sourceMapDetailLevel; options.sourceMapFormat = config.sourceMapFormat; options.sourceMapLocationMappings = config.sourceMapLocationMappings; + options.applyInputSourceMaps = config.applyInputSourceMaps; ImmutableMap.Builder inputSourceMaps = new ImmutableMap.Builder<>(); @@ -2245,6 +2246,17 @@ public CommandLineConfig setSourceMapLocationMappings( return this; } + private boolean applyInputSourceMaps = false; + + /** + * Whether to apply input source maps to the output, i.e. map back to original inputs from + * input files that have source maps applied to them. + */ + public CommandLineConfig setApplyInputSourceMaps(boolean applyInputSourceMaps) { + this.applyInputSourceMaps = applyInputSourceMaps; + return this; + } + private ArrayList> warningGuards = new ArrayList<>(); /** diff --git a/src/com/google/javascript/jscomp/CommandLineRunner.java b/src/com/google/javascript/jscomp/CommandLineRunner.java index 3c1c37a1858..c98fd099cf0 100644 --- a/src/com/google/javascript/jscomp/CommandLineRunner.java +++ b/src/com/google/javascript/jscomp/CommandLineRunner.java @@ -186,11 +186,11 @@ private static class Flags { @Option(name = "--js", handler = JsOptionHandler.class, - usage = "The JavaScript filename. You may specify multiple. " + - "The flag name is optional, because args are interpreted as files by default. " + - "You may also use minimatch-style glob patterns. For example, use " + - "--js='**.js' --js='!**_test.js' to recursively include all " + - "js files that do not end in _test.js") + usage = "The JavaScript filename. You may specify multiple. " + + "The flag name is optional, because args are interpreted as files by default. " + + "You may also use minimatch-style glob patterns. For example, use " + + "--js='**.js' --js='!**_test.js' to recursively include all " + + "js files that do not end in _test.js") private List js = new ArrayList<>(); @Option(name = "--jszip", @@ -200,8 +200,8 @@ private static class Flags { private List jszip = new ArrayList<>(); @Option(name = "--js_output_file", - usage = "Primary output filename. If not specified, output is " + - "written to stdout") + usage = "Primary output filename. If not specified, output is " + + "written to stdout") private String jsOutputFile = ""; @Option(name = "--module", @@ -279,30 +279,37 @@ private static class Flags { private String moduleOutputPathPrefix = "./"; @Option(name = "--create_source_map", - usage = "If specified, a source map file mapping the generated " + - "source files back to the original source file will be " + - "output to the specified path. The %outname% placeholder will " + - "expand to the name of the output file that the source map " + - "corresponds to.") + usage = "If specified, a source map file mapping the generated " + + "source files back to the original source file will be " + + "output to the specified path. The %outname% placeholder will " + + "expand to the name of the output file that the source map " + + "corresponds to.") private String createSourceMap = ""; @Option(name = "--source_map_format", hidden = true, - usage = "The source map format to produce. " + - "Options are V3 and DEFAULT, which are equivalent.") + usage = "The source map format to produce. " + + "Options are V3 and DEFAULT, which are equivalent.") private SourceMap.Format sourceMapFormat = SourceMap.Format.DEFAULT; @Option(name = "--source_map_location_mapping", - usage = "Source map location mapping separated by a '|' " + - "(i.e. filesystem-path|webserver-path)") + usage = "Source map location mapping separated by a '|' " + + "(i.e. filesystem-path|webserver-path)") private List sourceMapLocationMapping = new ArrayList<>(); @Option(name = "--source_map_input", hidden = true, - usage = "Source map locations for input files, separated by a '|', " + - "(i.e. input-file-path|input-source-map)") + usage = "Source map locations for input files, separated by a '|', " + + "(i.e. input-file-path|input-source-map)") private List sourceMapInputs = new ArrayList<>(); + @Option(name = "--apply_input_source_maps", + handler = BooleanOptionHandler.class, + hidden = true, + usage = "Whether to apply input source maps to the output source map, " + + "i.e. have the result map back to original inputs") + private boolean applyInputSourceMaps = false; + // Used to define the flag, values are stored by the handler. @SuppressWarnings("unused") @Option( @@ -338,24 +345,24 @@ private static class Flags { @Option(name = "--define", aliases = {"--D", "-D"}, - usage = "Override the value of a variable annotated @define. " + - "The format is [=], where is the name of a @define " + - "variable and is a boolean, number, or a single-quoted string " + - "that contains no single quotes. If [=] is omitted, " + - "the variable is marked true") + usage = "Override the value of a variable annotated @define. " + + "The format is [=], where is the name of a @define " + + "variable and is a boolean, number, or a single-quoted string " + + "that contains no single quotes. If [=] is omitted, " + + "the variable is marked true") private List define = new ArrayList<>(); @Option(name = "--charset", - usage = "Input and output charset for all files. By default, we " + - "accept UTF-8 as input and output US_ASCII") + usage = "Input and output charset for all files. By default, we " + + "accept UTF-8 as input and output US_ASCII") private String charset = ""; @Option(name = "--compilation_level", aliases = {"-O"}, - usage = "Specifies the compilation level to use. Options: " + - "WHITESPACE_ONLY, " + - "SIMPLE, " + - "ADVANCED") + usage = "Specifies the compilation level to use. Options: " + + "WHITESPACE_ONLY, " + + "SIMPLE, " + + "ADVANCED") private String compilationLevel = "SIMPLE"; private CompilationLevel compilationLevelParsed = null; @@ -367,9 +374,9 @@ private static class Flags { @Option(name = "--use_types_for_optimization", handler = BooleanOptionHandler.class, - usage = "Enable or disable the optimizations " + - "based on available type information. Inaccurate type annotations " + - "may result in incorrect results.") + usage = "Enable or disable the optimizations " + + "based on available type information. Inaccurate type annotations " + + "may result in incorrect results.") private boolean useTypesForOptimization = true; @Option(name = "--assume_function_wrapper", @@ -382,8 +389,8 @@ private static class Flags { @Option(name = "--warning_level", aliases = {"-W"}, - usage = "Specifies the warning level to use. Options: " + - "QUIET, DEFAULT, VERBOSE") + usage = "Specifies the warning level to use. Options: " + + "QUIET, DEFAULT, VERBOSE") private WarningLevel warningLevel = WarningLevel.DEFAULT; @Option(name = "--debug", @@ -542,9 +549,9 @@ private static class Flags { @Option(name = "--translations_project", hidden = true, - usage = "Scopes all translations to the specified project." + - "When specified, we will use different message ids so that messages " + - "in different projects can have different translations.") + usage = "Scopes all translations to the specified project." + + "When specified, we will use different message ids so that messages " + + "in different projects can have different translations.") private String translationsProject = null; @Option(name = "--flagfile", @@ -553,9 +560,9 @@ private static class Flags { private String flagFile = ""; @Option(name = "--warnings_whitelist_file", - usage = "A file containing warnings to suppress. Each line should be " + - "of the form\n" + - ":? ") + usage = "A file containing warnings to suppress. Each line should be " + + "of the form\n" + + ":? ") private String warningsWhitelistFile = ""; @Option(name = "--hide_warnings_for", @@ -569,8 +576,8 @@ private static class Flags { @Option(name = "--tracer_mode", hidden = true, - usage = "Shows the duration of each compiler pass and the impact to " + - "the compiled output size. Options: ALL, RAW_SIZE, TIMING_ONLY, OFF") + usage = "Shows the duration of each compiler pass and the impact to " + + "the compiled output size. Options: ALL, RAW_SIZE, TIMING_ONLY, OFF") private CompilerOptions.TracerMode tracerMode = CompilerOptions.TracerMode.OFF; @@ -1273,6 +1280,7 @@ private void initConfigFromFlags(String[] args, PrintStream out, PrintStream err List> mixedSources = null; List mappings = null; ImmutableMap sourceMapInputs = null; + boolean applyInputSourceMaps = false; try { flags.parse(processedArgs); @@ -1285,6 +1293,7 @@ private void initConfigFromFlags(String[] args, PrintStream out, PrintStream err mixedSources = flags.getMixedJsSources(); mappings = flags.getSourceMapLocationMappings(); sourceMapInputs = flags.getSourceMapInputs(); + applyInputSourceMaps = flags.applyInputSourceMaps; } catch (CmdLineException e) { reportError(e.getMessage()); } catch (IOException ioErr) { @@ -1419,6 +1428,7 @@ private void initConfigFromFlags(String[] args, PrintStream out, PrintStream err .setSourceMapFormat(flags.sourceMapFormat) .setSourceMapLocationMappings(mappings) .setSourceMapInputFiles(sourceMapInputs) + .setApplyInputSourceMaps(applyInputSourceMaps) .setWarningGuards(Flags.guardLevels) .setDefine(flags.define) .setCharset(flags.charset) diff --git a/src/com/google/javascript/jscomp/Compiler.java b/src/com/google/javascript/jscomp/Compiler.java index 485fcc2284f..5aa8d7414d4 100644 --- a/src/com/google/javascript/jscomp/Compiler.java +++ b/src/com/google/javascript/jscomp/Compiler.java @@ -80,7 +80,7 @@ * window, document. * */ -public class Compiler extends AbstractCompiler implements ErrorHandler { +public class Compiler extends AbstractCompiler implements ErrorHandler, SourceFileMapping { static final String SINGLETON_MODULE_NAME = "$singleton$"; static final DiagnosticType MODULE_DEPENDENCY_ERROR = @@ -487,6 +487,9 @@ private void initBasedOnOptions() { if (options.sourceMapOutputPath != null) { sourceMap = options.sourceMapFormat.getInstance(); sourceMap.setPrefixMappings(options.sourceMapLocationMappings); + if (options.applyInputSourceMaps) { + sourceMap.setSourceFileMapping(this); + } } } diff --git a/src/com/google/javascript/jscomp/CompilerOptions.java b/src/com/google/javascript/jscomp/CompilerOptions.java index 6c38bfd24f9..f6d3139268c 100644 --- a/src/com/google/javascript/jscomp/CompilerOptions.java +++ b/src/com/google/javascript/jscomp/CompilerOptions.java @@ -948,6 +948,12 @@ public void setTracerMode(TracerMode mode) { public SourceMap.Format sourceMapFormat = SourceMap.Format.DEFAULT; + /** + * Whether to apply input source maps to the output, i.e. map back to original inputs from + * input files that have source maps applied to them. + */ + boolean applyInputSourceMaps = false; + public List sourceMapLocationMappings = Collections.emptyList(); @@ -2456,6 +2462,10 @@ public void setSourceMapOutputPath(String sourceMapOutputPath) { this.sourceMapOutputPath = sourceMapOutputPath; } + public void setApplyInputSourceMaps(boolean applyInputSourceMaps) { + this.applyInputSourceMaps = applyInputSourceMaps; + } + public void setSourceMapIncludeSourcesContent(boolean sourceMapIncludeSourcesContent) { this.sourceMapIncludeSourcesContent = sourceMapIncludeSourcesContent; } diff --git a/src/com/google/javascript/jscomp/SourceFileMapping.java b/src/com/google/javascript/jscomp/SourceFileMapping.java new file mode 100644 index 00000000000..205b101546b --- /dev/null +++ b/src/com/google/javascript/jscomp/SourceFileMapping.java @@ -0,0 +1,36 @@ +/* + * Copyright 2016 The Closure Compiler Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.javascript.jscomp; + +import com.google.debugging.sourcemap.proto.Mapping.OriginalMapping; +import javax.annotation.Nullable; + +/** + * A SourceFileMapping maps a source file, line, and column into an {@link OriginalMapping}. + * + * @see com.google.debugging.sourcemap.SourceMapping + */ +public interface SourceFileMapping { + /** + * Returns the original mapping for the file name, line number and column position found + * in the source map. Returns {@code null} if none is found. + * + * @param lineNo The line number, 1-based. + * @param columnNo The column index, 1-based. + */ + @Nullable + OriginalMapping getSourceMapping(String fileName, int lineNo, int columnNo); +} diff --git a/src/com/google/javascript/jscomp/SourceMap.java b/src/com/google/javascript/jscomp/SourceMap.java index 65b2dd60788..6263155a3f6 100644 --- a/src/com/google/javascript/jscomp/SourceMap.java +++ b/src/com/google/javascript/jscomp/SourceMap.java @@ -21,6 +21,7 @@ import com.google.debugging.sourcemap.SourceMapFormat; import com.google.debugging.sourcemap.SourceMapGenerator; import com.google.debugging.sourcemap.SourceMapGeneratorFactory; +import com.google.debugging.sourcemap.proto.Mapping.OriginalMapping; import com.google.javascript.rhino.Node; import java.io.IOException; @@ -30,6 +31,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import javax.annotation.Nullable; /** * Collects information mapping the generated (compiled) source back to @@ -113,6 +115,13 @@ public String toString() { private List prefixMappings = Collections.emptyList(); private final Map sourceLocationFixupCache = new HashMap<>(); + /** + * A mapping derived from input source maps. Maps back to input sources that inputs to this + * compilation job have been generated from, and used to create a source map that maps all the way + * back to original inputs. {@code null} if no such mapping is wanted. + */ + @Nullable + private SourceFileMapping mapping; private SourceMap(SourceMapGenerator generator) { this.generator = generator; @@ -131,6 +140,17 @@ public void addMapping( return; } + int lineNo = node.getLineno(); + int charNo = node.getCharno(); + if (mapping != null) { + OriginalMapping sourceMapping = mapping.getSourceMapping(sourceFile, lineNo, charNo); + if (sourceMapping != null) { + sourceFile = sourceMapping.getOriginalFile(); + lineNo = sourceMapping.getLineNumber(); + charNo = sourceMapping.getColumnPosition(); + } + } + sourceFile = fixupSourceLocation(sourceFile); String originalName = node.getOriginalName(); @@ -141,7 +161,7 @@ public void addMapping( generator.addMapping( sourceFile, originalName, - new FilePosition(node.getLineno() - lineBaseOffset, node.getCharno()), + new FilePosition(lineNo - lineBaseOffset, charNo), outputStartPosition, outputEndPosition); } @@ -212,4 +232,8 @@ public void validate(boolean validate) { public void setPrefixMappings(List sourceMapLocationMappings) { this.prefixMappings = sourceMapLocationMappings; } + + public void setSourceFileMapping(SourceFileMapping mapping) { + this.mapping = mapping; + } } diff --git a/src/com/google/javascript/jscomp/ant/CompileTask.java b/src/com/google/javascript/jscomp/ant/CompileTask.java index 30a06e23a89..53b06df4d34 100644 --- a/src/com/google/javascript/jscomp/ant/CompileTask.java +++ b/src/com/google/javascript/jscomp/ant/CompileTask.java @@ -99,6 +99,7 @@ public final class CompileTask private String sourceMapFormat; private File sourceMapOutputFile; private String sourceMapLocationMapping; + private boolean applyInputSourceMaps; public CompileTask() { this.languageIn = CompilerOptions.LanguageMode.ECMASCRIPT6; @@ -487,6 +488,8 @@ private CompilerOptions createCompilerOptions() { options.setSourceMapLocationMappings(Arrays.asList(lm)); } + options.setApplyInputSourceMaps(applyInputSourceMaps); + if (sourceMapOutputFile != null) { File parentFile = sourceMapOutputFile.getParentFile(); if (parentFile.mkdirs()) { diff --git a/test/com/google/javascript/jscomp/CompilerTest.java b/test/com/google/javascript/jscomp/CompilerTest.java index 6ae0af88164..2052c72f5f7 100644 --- a/test/com/google/javascript/jscomp/CompilerTest.java +++ b/test/com/google/javascript/jscomp/CompilerTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.debugging.sourcemap.FilePosition; +import com.google.debugging.sourcemap.SourceMapConsumerV3; import com.google.debugging.sourcemap.SourceMapGeneratorV3; import com.google.debugging.sourcemap.proto.Mapping.OriginalMapping; import com.google.javascript.rhino.InputId; @@ -34,6 +35,7 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.StringWriter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -198,6 +200,36 @@ public SourceFile apply(String filename) { }; } + public void testApplyInputSourceMaps() throws Exception { + FilePosition originalSourcePosition = new FilePosition(17, 25); + ImmutableMap inputSourceMaps = ImmutableMap.of( + "input.js", + sourcemap( + "input.js.map", + "input.ts", + originalSourcePosition)); + + CompilerOptions options = new CompilerOptions(); + options.sourceMapOutputPath = "fake/source_map_path.js.map"; + options.inputSourceMaps = inputSourceMaps; + options.applyInputSourceMaps = true; + Compiler compiler = new Compiler(); + compiler.compile(EMPTY_EXTERNS.get(0), + SourceFile.fromCode("input.js", "// Unmapped line\nvar x = 1;\nalert(x);"), options); + assertThat(compiler.toSource()).isEqualTo("var x=1;alert(x);"); + SourceMap sourceMap = compiler.getSourceMap(); + StringWriter out = new StringWriter(); + sourceMap.appendTo(out, "source.js.map"); + SourceMapConsumerV3 consumer = new SourceMapConsumerV3(); + consumer.parse(out.toString()); + // Column 5 contains the first actually mapped code ('x'). + OriginalMapping mapping = consumer.getMappingForLine(1, 5); + assertThat(mapping.getOriginalFile()).isEqualTo("input.ts"); + // FilePosition above is 0-based, whereas OriginalMapping is 1-based, thus 18 & 26. + assertThat(mapping.getLineNumber()).isEqualTo(18); + assertThat(mapping.getColumnPosition()).isEqualTo(26); + } + private Compiler initCompilerForCommonJS( List inputs, List entryPoints) throws Exception {