Teach deps.JsFileParser to understand ES6 modules.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=129388646

src/com/google/javascript/jscomp/LazyParsedDependencyInfo.java

@@ -19,8 +19,8 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.javascript.jscomp.deps.DependencyInfo;
+import com.google.javascript.jscomp.deps.ModuleLoader;
 import com.google.javascript.jscomp.parsing.parser.FeatureSet;
-
 import java.util.Collection;
 import java.util.Map;
 import java.util.TreeMap;
@@ -30,9 +30,6 @@
  */
 public class LazyParsedDependencyInfo implements DependencyInfo {
 
-  static final DiagnosticType MODULE_CONFLICT = DiagnosticType.warning(
-      "JSC_MODULE_CONFLICT", "File has both goog.module and ES6 modules: {0}");
-
   private final DependencyInfo delegate;
   private final JsAst ast;
   private final AbstractCompiler compiler;
@@ -52,8 +49,9 @@ public ImmutableMap<String, String> getLoadFlags() {
       loadFlagsBuilder.putAll(delegate.getLoadFlags());
       FeatureSet features = ((JsAst) ast).getFeatures(compiler);
       if (features.hasEs6Modules()) {
-        if (loadFlagsBuilder.containsKey("module")) {
-          compiler.report(JSError.make(MODULE_CONFLICT, getName()));
+        String previousModule = loadFlagsBuilder.get("module");
+        if (previousModule != null && !previousModule.equals("es6")) {
+          compiler.report(JSError.make(ModuleLoader.MODULE_CONFLICT, getName()));
         }
         loadFlagsBuilder.put("module", "es6");
       }

src/com/google/javascript/jscomp/deps/DepsGenerator.java

@@ -31,7 +31,6 @@
 import com.google.javascript.jscomp.JsAst;
 import com.google.javascript.jscomp.LazyParsedDependencyInfo;
 import com.google.javascript.jscomp.SourceFile;
-
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
@@ -68,6 +67,7 @@ public static enum InclusionStrategy {
   private final Collection<SourceFile> deps;
   private final String closurePathAbs;
   private final InclusionStrategy mergeStrategy;
+  private final ModuleLoader loader;
   final ErrorManager errorManager;
 
   static final DiagnosticType SAME_FILE_WARNING = DiagnosticType.warning(
@@ -102,12 +102,14 @@ public DepsGenerator(
       Collection<SourceFile> srcs,
       InclusionStrategy mergeStrategy,
       String closurePathAbs,
-      ErrorManager errorManager) {
+      ErrorManager errorManager,
+      ModuleLoader loader) {
     this.deps = deps;
     this.srcs = srcs;
     this.mergeStrategy = mergeStrategy;
     this.closurePathAbs = closurePathAbs;
     this.errorManager = errorManager;
+    this.loader = loader;
   }
 
   /**
@@ -323,7 +325,7 @@ private Map<String, DependencyInfo> parseDepsFiles() throws IOException {
   private Map<String, DependencyInfo> parseSources(
       Set<String> preparsedFiles) throws IOException {
     Map<String, DependencyInfo> parsedFiles = new HashMap<>();
-    JsFileParser jsParser = new JsFileParser(errorManager);
+    JsFileParser jsParser = new JsFileParser(errorManager).setModuleLoader(loader);
     Compiler compiler = new Compiler();
     compiler.init(
         ImmutableList.<SourceFile>of(), ImmutableList.<SourceFile>of(), new CompilerOptions());

src/com/google/javascript/jscomp/deps/Es6SortedDependencies.java

@@ -18,11 +18,12 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.LinkedHashMultimap;
 import com.google.common.collect.Multimap;
-
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Deque;
 import java.util.HashMap;
@@ -142,7 +143,16 @@ private void orderInput(INPUT input) {
   private void processInputs() {
     // Index.
     for (INPUT userOrderedInput : userOrderedInputs) {
-      if (userOrderedInput.getProvides().isEmpty()) {
+      Collection<String> provides = userOrderedInput.getProvides();
+      String firstProvide = Iterables.getFirst(provides, null);
+      if (firstProvide == null
+          // TODO(sdh): It would be better to have a more robust way to distinguish
+          // between actual provided symbols and synthetic symbols generated for
+          // ES6 (or other) modules.  We can't read loadFlags here (to see if
+          // the module type is 'es6') either, since that requires a full parse.
+          // So for now we rely on the heuristic that all generated provides start
+          // with "module$".
+          || (provides.size() == 1 && firstProvide.startsWith("module$"))) {
         nonExportingInputs.put(
             ModuleNames.fileToModuleName(userOrderedInput.getName()), userOrderedInput);
       }

src/com/google/javascript/jscomp/deps/JsFileParser.java

@@ -18,51 +18,91 @@
 
 import com.google.common.annotations.GwtIncompatible;
 import com.google.common.base.CharMatcher;
+import com.google.javascript.jscomp.CheckLevel;
 import com.google.javascript.jscomp.ErrorManager;
-
+import com.google.javascript.jscomp.JSError;
 import java.io.Reader;
 import java.io.StringReader;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
- * A parser that can extract goog.require() and goog.provide() dependency
- * information from a .js file.
+ * A parser that can extract dependency information from a .js file, including
+ * goog.require, goog.provide, goog.module, import statements, and export statements.
  *
  * @author agrieve@google.com (Andrew Grieve)
  */
 @GwtIncompatible("java.util.regex")
 public final class JsFileParser extends JsFileLineParser {
 
-  private static Logger logger = Logger.getLogger(JsFileParser.class.getName());
+  private static final Logger logger = Logger.getLogger(JsFileParser.class.getName());
 
   /** Pattern for matching goog.provide(*) and goog.require(*). */
   private static final Pattern GOOG_PROVIDE_REQUIRE_PATTERN =
       Pattern.compile(
           "(?:^|;)(?:[a-zA-Z0-9$_,:{}\\s]+=)?\\s*"
               + "goog\\.(provide|module|require|addDependency)\\s*\\((.*?)\\)");
 
+  /**
+   * Pattern for matching import ... from './path/to/file'.
+   *
+   * <p>Unlike the goog.require() pattern above, this pattern does not
+   * allow multiple statements per line.  The import/export <b>must</b>
+   * be at the beginning of the line to match.
+   */
+  private static final Pattern ES6_MODULE_PATTERN =
+      Pattern.compile(
+          // Require the import/export to be at the beginning of the line
+          "^"
+          // Either an import or export, but we don't care which, followed by at least one space
+          + "(?:import|export)\\b\\s*"
+          // Skip any identifier chars, as well as star, comma, braces, and spaces
+          // This should match, e.g., "* as foo from ", or "Foo, {Bar as Baz} from ".
+          // The 'from' keyword is required except in the case of "import '...';",
+          // where there's nothing between 'import' and the module key string literal.
+          + "(?:[a-zA-Z0-9$_*,{}\\s]+\\bfrom\\s*|)"
+          // Imports require a string literal at the end; it's optional for exports
+          // (e.g. "export * from './other';", which is effectively also an import).
+          // This optionally captures group #1, which is the imported module name.
+          + "(?:['\"]([^'\"]+)['\"])?"
+          // Finally, this should be the entire statement, so ensure there's a semicolon.
+          + "\\s*;");
+
+  /**
+   * Pattern for 'export' keyword, e.g. "export default class ..." or "export {blah}".
+   * The '\b' ensures we don't also match "exports = ...", which is not an ES6 module.
+   */
+  private static final Pattern ES6_EXPORT_PATTERN = Pattern.compile("^export\\b");
+
   /** The first non-comment line of base.js */
   private static final String BASE_JS_START = "var COMPILED = false;";
 
   /** The start of a bundled goog.module, i.e. one that is wrapped in a goog.loadModule call */
   private static final String BUNDLED_GOOG_MODULE_START = "goog.loadModule(function(";
 
   /** Matchers used in the parsing. */
-  private Matcher googMatcher = GOOG_PROVIDE_REQUIRE_PATTERN.matcher("");
+  private final Matcher googMatcher = GOOG_PROVIDE_REQUIRE_PATTERN.matcher("");
+
+  /** Matchers used in the parsing. */
+  private final Matcher es6Matcher = ES6_MODULE_PATTERN.matcher("");
 
   /** The info for the file we are currently parsing. */
   private List<String> provides;
   private List<String> requires;
   private boolean fileHasProvidesOrRequires;
+  private ModuleLoader loader = ModuleLoader.EMPTY;
+  private ModuleLoader.ModuleUri fileUri;
 
   private enum ModuleType {
     NON_MODULE,
     UNWRAPPED_GOOG_MODULE,
     WRAPPED_GOOG_MODULE,
+    ES6_MODULE,
   }
 
   private ModuleType moduleType;
@@ -97,6 +137,17 @@ public JsFileParser setIncludeGoogBase(boolean include) {
     return this;
   }
 
+  /**
+   * Sets a list of "module root" URIs, which allow relativizing filenames
+   * for modules.
+   *
+   * @return this for easy chaining.
+   */
+  public JsFileParser setModuleLoader(ModuleLoader loader) {
+    this.loader = loader;
+    return this;
+  }
+
   /**
    * Parses the given file and returns the dependency information that it
    * contained.
@@ -115,21 +166,46 @@ public DependencyInfo parseFile(String filePath, String closureRelativePath,
 
   private DependencyInfo parseReader(String filePath,
       String closureRelativePath, Reader fileContents) {
-    provides = new ArrayList<>();
-    requires = new ArrayList<>();
-    fileHasProvidesOrRequires = false;
-    moduleType = ModuleType.NON_MODULE;
+    this.provides = new ArrayList<>();
+    this.requires = new ArrayList<>();
+    this.fileHasProvidesOrRequires = false;
+    this.fileUri = loader.resolve(filePath);
+    this.moduleType = ModuleType.NON_MODULE;
 
     logger.fine("Parsing Source: " + filePath);
     doParse(filePath, fileContents);
 
+    if (moduleType == ModuleType.ES6_MODULE) {
+      provides.add(fileUri.toModuleName());
+    }
+
+    Map<String, String> loadFlags = new LinkedHashMap<>();
+    switch (moduleType) {
+      case UNWRAPPED_GOOG_MODULE:
+        loadFlags.put("module", "goog");
+        break;
+      case ES6_MODULE:
+        loadFlags.put("module", "es6");
+        break;
+      default:
+        // Nothing to do here.
+    }
+
     DependencyInfo dependencyInfo = new SimpleDependencyInfo(
-        closureRelativePath, filePath, provides, requires,
-        moduleType == ModuleType.UNWRAPPED_GOOG_MODULE);
+        closureRelativePath, filePath, provides, requires, loadFlags);
     logger.fine("DepInfo: " + dependencyInfo);
     return dependencyInfo;
   }
 
+  private void setModuleType(ModuleType type) {
+    if (moduleType != type && moduleType != ModuleType.NON_MODULE) {
+      // TODO(sdh): should this be an error?
+      errorManager.report(
+          CheckLevel.WARNING, JSError.make(ModuleLoader.MODULE_CONFLICT, fileUri.toString()));
+    }
+    moduleType = type;
+  }
+
   /**
    * Parses a line of JavaScript, extracting goog.provide and goog.require
    * information.
@@ -161,7 +237,7 @@ protected boolean parseLine(String line) throws ParseException {
         boolean isRequire = firstChar == 'r';
 
         if (isModule && this.moduleType != ModuleType.WRAPPED_GOOG_MODULE) {
-          this.moduleType = ModuleType.UNWRAPPED_GOOG_MODULE;
+          setModuleType(ModuleType.UNWRAPPED_GOOG_MODULE);
         }
 
         if (isProvide || isRequire) {
@@ -187,7 +263,29 @@ protected boolean parseLine(String line) throws ParseException {
       // base.js can't provide or require anything else.
       return false;
     } else if (line.startsWith(BUNDLED_GOOG_MODULE_START)) {
-      this.moduleType = ModuleType.WRAPPED_GOOG_MODULE;
+      setModuleType(ModuleType.WRAPPED_GOOG_MODULE);
+    }
+
+    if (line.startsWith("import") || line.startsWith("export")) {
+      es6Matcher.reset(line);
+      while (es6Matcher.find()) {
+        setModuleType(ModuleType.ES6_MODULE);
+        lineHasProvidesOrRequires = true;
+
+        String arg = es6Matcher.group(1);
+        if (arg != null) {
+          if (arg.startsWith("goog:")) {
+            requires.add(arg.substring(5)); // cut off the "goog:" prefix
+          } else {
+            requires.add(fileUri.resolveEs6Module(arg).toModuleName());
+          }
+        }
+      }
+
+      // This check is only relevant for modules that don't import anything.
+      if (moduleType != ModuleType.ES6_MODULE && ES6_EXPORT_PATTERN.matcher(line).lookingAt()) {
+        setModuleType(ModuleType.ES6_MODULE);
+      }
     }
 
     return !shortcutMode || lineHasProvidesOrRequires