Adds support for webpack.

Adds a webpack specific module resolution mode to lookup modules by their webpack id.
Adds support for dynamically loaded modules by treating them as entry points in the module graph.
ProcessCommonJSModules is updated to recognize and rewrite webpack specific require calls.

Closes #2762

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=182133181

src/com/google/javascript/jscomp/AbstractCommandLineRunner.java

@@ -1085,6 +1085,7 @@ protected int doRun() throws IOException {
 
       ImmutableMap.Builder<String, SourceMapInput> inputSourceMaps
           = new ImmutableMap.Builder<>();
+      ImmutableMap.Builder<String, String> inputPathByWebpackId = new ImmutableMap.Builder<>();
 
       boolean foundJsonInputSourceMap = false;
       for (JsonFileSpec jsonFile : jsonFiles) {
@@ -1095,12 +1096,20 @@ protected int doRun() throws IOException {
           inputSourceMaps.put(jsonFile.getPath(), new SourceMapInput(sourceMap));
           foundJsonInputSourceMap = true;
         }
+        if (jsonFile.getWebpackId() != null) {
+          inputPathByWebpackId.put(jsonFile.getWebpackId(), jsonFile.getPath());
+        }
       }
 
       if (foundJsonInputSourceMap) {
         inputSourceMaps.putAll(options.inputSourceMaps);
         options.inputSourceMaps = inputSourceMaps.build();
       }
+
+      compiler.initWebpackMap(inputPathByWebpackId.build());
+    } else {
+      ImmutableMap<String, String> emptyMap = ImmutableMap.of();
+      compiler.initWebpackMap(emptyMap);
     }
 
     compiler.initWarningsGuard(options.getWarningsGuard());
@@ -2708,15 +2717,22 @@ protected static class JsonFileSpec {
     private final String src;
     private final String path;
     private String sourceMap;
+    @Nullable
+    private final String webpackId;
 
     public JsonFileSpec(String src, String path) {
-      this(src, path, null);
+      this(src, path, null, null);
     }
 
     public JsonFileSpec(String src, String path, String sourceMap) {
+      this(src, path, sourceMap, null);
+    }
+
+    public JsonFileSpec(String src, String path, String sourceMap, @Nullable String webpackId) {
       this.src = src;
       this.path = path;
       this.sourceMap = sourceMap;
+      this.webpackId = webpackId;
     }
 
     public String getSrc() {
@@ -2731,6 +2747,10 @@ public String getSourceMap() {
       return this.sourceMap;
     }
 
+    public String getWebpackId() {
+      return this.webpackId;
+    }
+
     public void setSourceMap(String map) {
       this.sourceMap = map;
     }

src/com/google/javascript/jscomp/ClosureRewriteModule.java

@@ -30,6 +30,7 @@
 import com.google.common.collect.Multimap;
 import com.google.javascript.jscomp.NodeTraversal.Callback;
 import com.google.javascript.jscomp.NodeTraversal.ScopedCallback;
+import com.google.javascript.jscomp.deps.ModuleLoader.ResolutionMode;
 import com.google.javascript.rhino.IR;
 import com.google.javascript.rhino.JSDocInfo;
 import com.google.javascript.rhino.JSDocInfoBuilder;
@@ -981,7 +982,9 @@ private void recordGoogModuleGet(NodeTraversal t, Node call) {
       t.report(legacyNamespaceNode, INVALID_GET_NAMESPACE);
       return;
     }
-    if (!currentScript.isModule && t.inGlobalScope()) {
+    if (!currentScript.isModule
+        && t.inGlobalScope()
+        && compiler.getOptions().moduleResolutionMode != ResolutionMode.WEBPACK) {
       t.report(legacyNamespaceNode, INVALID_GET_CALL_SCOPE);
       return;
     }

src/com/google/javascript/jscomp/CommandLineRunner.java

@@ -779,7 +779,7 @@ private static class Flags {
       usage =
           "Specifies how the compiler locates modules. BROWSER requires all module imports "
               + "to begin with a '.' or '/' and have a file extension. NODE uses the node module "
-              + "rules."
+              + "rules. WEBPACK looks up modules from a special lookup map."
     )
     private ModuleLoader.ResolutionMode moduleResolutionMode = ModuleLoader.ResolutionMode.BROWSER;
 

src/com/google/javascript/jscomp/Compiler.java

@@ -171,6 +171,8 @@ public class Compiler extends AbstractCompiler implements ErrorHandler, SourceFi
 
   private transient IncrementalScopeCreator scopeCreator = null;
 
+  private ImmutableMap<String, String> inputPathByWebpackId;
+
   /**
    * Subclasses are responsible for loading sources that were not provided as explicit inputs to the
    * compiler. For example, looking up sources referenced within sourcemaps.
@@ -1760,7 +1762,7 @@ Node parseInputs() {
                 inputs,
                 ModuleLoader.PathResolver.RELATIVE,
                 options.moduleResolutionMode,
-                null);
+                inputPathByWebpackId);
 
         if (options.moduleResolutionMode == ModuleLoader.ResolutionMode.NODE) {
           // processJsonInputs requires a module loader to already be defined
@@ -2000,7 +2002,8 @@ private List<CompilerInput> depthFirstDependenciesFromInput(
     }
 
     FindModuleDependencies findDeps =
-        new FindModuleDependencies(this, supportEs6Modules, supportCommonJSModules);
+        new FindModuleDependencies(
+            this, supportEs6Modules, supportCommonJSModules, inputPathByWebpackId);
     findDeps.process(input.getAstRoot(this));
 
     // If this input was imported by another module, it is itself a module
@@ -2010,7 +2013,10 @@ private List<CompilerInput> depthFirstDependenciesFromInput(
     }
     this.moduleTypesByName.put(input.getPath().toModuleName(), input.getJsModuleType());
 
-    for (String requiredNamespace : input.getRequires()) {
+    ArrayList<String> allDeps = new ArrayList<>();
+    allDeps.addAll(input.getRequires());
+    allDeps.addAll(input.getDynamicRequires());
+    for (String requiredNamespace : allDeps) {
       CompilerInput requiredInput = null;
       boolean requiredByModuleImport = false;
       if (inputsByProvide.containsKey(requiredNamespace)) {
@@ -3493,6 +3499,10 @@ private void renameModules(List<JSModule> newModules, List<JSModule> deserialize
     return;
   }
 
+  void initWebpackMap(ImmutableMap<String, String> inputPathByWebpackId) {
+    this.inputPathByWebpackId = inputPathByWebpackId;
+  }
+
   protected CompilerExecutor createCompilerExecutor() {
     return new CompilerExecutor();
   }

src/com/google/javascript/jscomp/CompilerInput.java

@@ -61,6 +61,7 @@ public class CompilerInput implements SourceAst, DependencyInfo {
   private final List<String> extraRequires = new ArrayList<>();
   private final List<String> extraProvides = new ArrayList<>();
   private final List<String> orderedRequires = new ArrayList<>();
+  private final List<String> dynamicRequires = new ArrayList<>();
   private boolean hasFullParseDependencyInfo = false;
   private ModuleType jsModuleType = ModuleType.NONE;
 
@@ -219,6 +220,28 @@ public boolean addOrderedRequire(String require) {
     return false;
   }
 
+  /**
+   * Returns the types that this input dynamically depends on in the order seen in the file. The
+   * returned types were loaded dynamically so while they are part of the dependency graph, they do
+   * not need sorted before this input.
+   */
+  public ImmutableList<String> getDynamicRequires() {
+    return ImmutableList.copyOf(dynamicRequires);
+  }
+
+  /**
+   * Registers a type that this input depends on in the order seen in the file. The type was loaded
+   * dynamically so while it is part of the dependency graph, it does not need sorted before this
+   * input.
+   */
+  public boolean addDynamicRequire(String require) {
+    if (!dynamicRequires.contains(require)) {
+      dynamicRequires.add(require);
+      return true;
+    }
+    return false;
+  }
+
   public void setHasFullParseDependencyInfo(boolean hasFullParseDependencyInfo) {
     this.hasFullParseDependencyInfo = hasFullParseDependencyInfo;
   }

src/com/google/javascript/jscomp/DefaultPassConfig.java

@@ -3618,7 +3618,7 @@ protected CompilerPass create(AbstractCompiler compiler) {
 
         @Override
         public FeatureSet featureSet() {
-          return ES8_MODULES;
+          return ES_NEXT;
         }
       };
 

src/com/google/javascript/jscomp/FindModuleDependencies.java

@@ -18,6 +18,7 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.javascript.jscomp.CompilerInput.ModuleType;
 import com.google.javascript.jscomp.Es6RewriteModules.FindGoogProvideOrGoogModule;
 import com.google.javascript.jscomp.deps.ModuleLoader;
@@ -29,28 +30,35 @@
  * support a strict depth-first dependency ordering. Marks an input as providing its module name.
  *
  * <p>Discovers dependencies from:
+ *
  * <ul>
- *   <li> goog.require calls
- *   <li> ES6 import statements
- *   <li> CommonJS require statements
+ *   <li>goog.require calls
+ *   <li>ES6 import statements
+ *   <li>CommonJS require statements
  * </ul>
  *
  * <p>The order of dependency references is preserved so that a deterministic depth-first ordering
  * can be achieved.
  *
  * @author chadkillingsworth@gmail.com (Chad Killingsworth)
  */
-public class FindModuleDependencies extends NodeTraversal.AbstractPostOrderCallback {
+public class FindModuleDependencies implements NodeTraversal.ScopedCallback {
   private final AbstractCompiler compiler;
   private final boolean supportsEs6Modules;
   private final boolean supportsCommonJsModules;
   private ModuleType moduleType = ModuleType.NONE;
+  private Scope dynamicImportScope = null;
+  private final ImmutableMap<String, String> inputPathByWebpackId;
 
   FindModuleDependencies(
-      AbstractCompiler compiler, boolean supportsEs6Modules, boolean supportsCommonJsModules) {
+      AbstractCompiler compiler,
+      boolean supportsEs6Modules,
+      boolean supportsCommonJsModules,
+      ImmutableMap<String, String> inputPathByWebpackId) {
     this.compiler = compiler;
     this.supportsEs6Modules = supportsEs6Modules;
     this.supportsCommonJsModules = supportsCommonJsModules;
+    this.inputPathByWebpackId = inputPathByWebpackId;
   }
 
   public void process(Node root) {
@@ -73,14 +81,34 @@ public void process(Node root) {
 
     if (moduleType == ModuleType.ES6) {
       convertToEs6Module(root, true);
+    } else if (moduleType == ModuleType.NONE
+        && inputPathByWebpackId != null
+        && inputPathByWebpackId.containsValue(input.getPath().toString())) {
+      moduleType = ModuleType.IMPORTED_SCRIPT;
     }
+
     input.addProvide(input.getPath().toModuleName());
     input.setJsModuleType(moduleType);
     input.setHasFullParseDependencyInfo(true);
   }
 
+  @Override
+  public boolean shouldTraverse(NodeTraversal t, Node n, Node parent) {
+    if (supportsCommonJsModules
+        && n.isFunction()
+        && ProcessCommonJSModules.isCommonJsDynamicImportCallback(
+            n, compiler.getOptions().moduleResolutionMode)) {
+      if (dynamicImportScope == null) {
+        dynamicImportScope = t.getScope();
+      }
+    }
+
+    return true;
+  }
+
   @Override
   public void visit(NodeTraversal t, Node n, Node parent) {
+    ModuleLoader.ResolutionMode resolutionMode = compiler.getOptions().moduleResolutionMode;
     if (parent == null
         || NodeUtil.isControlStructure(parent)
         || NodeUtil.isStatementBlock(parent)) {
@@ -108,18 +136,31 @@ public void visit(NodeTraversal t, Node n, Node parent) {
       moduleType = ModuleType.ES6;
       addEs6ModuleImportToGraph(t, n);
     } else if (supportsCommonJsModules) {
-      if (moduleType != ModuleType.GOOG && ProcessCommonJSModules.isCommonJsExport(t, n)) {
+      if (moduleType != ModuleType.GOOG
+          && ProcessCommonJSModules.isCommonJsExport(t, n, resolutionMode)) {
         moduleType = ModuleType.COMMONJS;
-      } else if (ProcessCommonJSModules.isCommonJsImport(n)) {
-        String path = ProcessCommonJSModules.getCommonJsImportPath(n);
+      } else if (ProcessCommonJSModules.isCommonJsImport(n, resolutionMode)) {
+        String path = ProcessCommonJSModules.getCommonJsImportPath(n, resolutionMode);
 
         ModuleLoader.ModulePath modulePath =
             t.getInput()
                 .getPath()
                 .resolveJsModule(path, n.getSourceFileName(), n.getLineno(), n.getCharno());
 
         if (modulePath != null) {
-          t.getInput().addOrderedRequire(modulePath.toModuleName());
+          if (dynamicImportScope != null
+              || (n.getParent().isCall()
+                  && n.getPrevious() != null
+                  && n.getPrevious().isGetProp()
+                  && n.getPrevious().getFirstChild().isCall()
+                  && n.getPrevious().getFirstFirstChild().isQualifiedName()
+                  && n.getPrevious()
+                      .getFirstFirstChild()
+                      .matchesQualifiedName("__webpack_require__.e"))) {
+            t.getInput().addDynamicRequire(modulePath.toModuleName());
+          } else {
+            t.getInput().addOrderedRequire(modulePath.toModuleName());
+          }
         }
       }
 
@@ -140,15 +181,14 @@ public void visit(NodeTraversal t, Node n, Node parent) {
     }
   }
 
-  /**
-   * Convert a script into a module by marking it's root node as a module body. This allows a script
-   * which is imported as a module to be scoped as a module even without "import" or "export"
-   * statements. Fails if the file contains a goog.provide or goog.module.
-   *
-   * @return True, if the file is now an ES6 module. False, if the file must remain a script.
-   */
-  public boolean convertToEs6Module(Node root) {
-    return this.convertToEs6Module(root, false);
+  @Override
+  public void enterScope(NodeTraversal t) {}
+
+  @Override
+  public void exitScope(NodeTraversal t) {
+    if (t.getScope() == dynamicImportScope) {
+      dynamicImportScope = null;
+    }
   }
 
   /**

src/com/google/javascript/jscomp/JSModuleGraph.java

@@ -412,8 +412,7 @@ public ImmutableList<CompilerInput> manageDependencies(
 
     SortedDependencies<CompilerInput> sorter = new Es6SortedDependencies<>(inputs);
 
-    Iterable<CompilerInput> entryPointInputs = createEntryPointInputs(
-        depOptions, inputs, sorter);
+    Set<CompilerInput> entryPointInputs = createEntryPointInputs(depOptions, inputs, sorter);
 
     HashMap<String, CompilerInput> inputsByProvide = new HashMap<>();
     for (CompilerInput input : inputs) {
@@ -426,6 +425,17 @@ public ImmutableList<CompilerInput> manageDependencies(
       }
     }
 
+    // Dynamically imported files must be added to the module graph, but
+    // they should not be ordered ahead of the files that import them.
+    // We add them as entry points to ensure they get included.
+    for (CompilerInput input : inputs) {
+      for (String require : input.getDynamicRequires()) {
+        if (inputsByProvide.containsKey(require)) {
+          entryPointInputs.add(inputsByProvide.get(require));
+        }
+      }
+    }
+
     // The order of inputs, sorted independently of modules.
     List<CompilerInput> absoluteOrder =
         sorter.getDependenciesOf(inputs, depOptions.shouldSortDependencies());
@@ -539,7 +549,7 @@ private List<CompilerInput> getDepthFirstDependenciesOf(
     return orderedInputs;
   }
 
-  private Collection<CompilerInput> createEntryPointInputs(
+  private Set<CompilerInput> createEntryPointInputs(
       DependencyOptions depOptions,
       List<CompilerInput> inputs,
       SortedDependencies<CompilerInput> sorter)