Support :require when using the flat classpath representation.

:require was re-incarnated in scala#4051,
it seems to be used by the spark repl. This commit makes it work when
using the flat classpath representation.

src/compiler/scala/tools/nsc/Global.scala

@@ -7,29 +7,29 @@ package scala
 package tools
 package nsc
 
-import java.io.{ File, IOException, FileNotFoundException }
+import java.io.{File, FileNotFoundException, IOException}
 import java.net.URL
-import java.nio.charset.{ Charset, CharsetDecoder, IllegalCharsetNameException, UnsupportedCharsetException }
-import scala.collection.{ mutable, immutable }
-import io.{ SourceReader, AbstractFile, Path }
+import java.nio.charset.{Charset, CharsetDecoder, IllegalCharsetNameException, UnsupportedCharsetException}
+import scala.collection.{immutable, mutable}
+import io.{AbstractFile, Path, SourceReader}
 import reporters.Reporter
-import util.{ ClassFileLookup, ClassPath, MergedClassPath, StatisticsInfo, returning }
+import util.{ClassFileLookup, ClassPath, StatisticsInfo, returning}
 import scala.reflect.ClassTag
-import scala.reflect.internal.util.{ ScalaClassLoader, SourceFile, NoSourceFile, BatchSourceFile, ScriptSourceFile }
+import scala.reflect.internal.util.{BatchSourceFile, NoSourceFile, ScalaClassLoader, ScriptSourceFile, SourceFile}
 import scala.reflect.internal.pickling.PickleBuffer
-import symtab.{ Flags, SymbolTable, SymbolTrackers }
+import symtab.{Flags, SymbolTable, SymbolTrackers}
 import symtab.classfile.Pickler
 import plugins.Plugins
 import ast._
 import ast.parser._
 import typechecker._
 import transform.patmat.PatternMatching
 import transform._
-import backend.{ ScalaPrimitives, JavaPlatform }
+import backend.{JavaPlatform, ScalaPrimitives}
 import backend.jvm.GenBCode
 import scala.language.postfixOps
 import scala.tools.nsc.ast.{TreeGen => AstTreeGen}
-import scala.tools.nsc.classpath.FlatClassPath
+import scala.tools.nsc.classpath._
 import scala.tools.nsc.settings.ClassPathRepresentationType
 
 class Global(var currentSettings: Settings, var reporter: Reporter)
@@ -102,9 +102,6 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
   type ThisPlatform = JavaPlatform { val global: Global.this.type }
   lazy val platform: ThisPlatform  = new GlobalPlatform
 
-  type PlatformClassPath = ClassPath[AbstractFile]
-  type OptClassPath = Option[PlatformClassPath]
-
   def classPath: ClassFileLookup[AbstractFile] = settings.YclasspathImpl.value match {
     case ClassPathRepresentationType.Flat => flatClassPath
     case ClassPathRepresentationType.Recursive => recursiveClassPath
@@ -771,13 +768,17 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
 
   /** Extend classpath of `platform` and rescan updated packages. */
   def extendCompilerClassPath(urls: URL*): Unit = {
-    if (settings.YclasspathImpl.value == ClassPathRepresentationType.Flat)
-      throw new UnsupportedOperationException("Flat classpath doesn't support extending the compiler classpath")
-
-    val newClassPath = platform.classPath.mergeUrlsIntoClassPath(urls: _*)
-    platform.currentClassPath = Some(newClassPath)
-    // Reload all specified jars into this compiler instance
-    invalidateClassPathEntries(urls.map(_.getPath): _*)
+    if (settings.YclasspathImpl.value == ClassPathRepresentationType.Flat) {
+      val urlClasspaths = urls.map(u => FlatClassPathFactory.newClassPath(AbstractFile.getURL(u), settings))
+      val newClassPath = AggregateFlatClassPath.createAggregate(platform.flatClassPath +: urlClasspaths : _*)
+      platform.currentFlatClassPath = Some(newClassPath)
+      invalidateClassPathEntries(urls.map(_.getPath): _*)
+    } else {
+      val newClassPath = platform.classPath.mergeUrlsIntoClassPath(urls: _*)
+      platform.currentClassPath = Some(newClassPath)
+      // Reload all specified jars into this compiler instance
+      invalidateClassPathEntries(urls.map(_.getPath): _*)
+    }
   }
 
   // ------------ Invalidations ---------------------------------
@@ -809,43 +810,60 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
    *                entries on the classpath.
    */
   def invalidateClassPathEntries(paths: String*): Unit = {
-    if (settings.YclasspathImpl.value == ClassPathRepresentationType.Flat)
-      throw new UnsupportedOperationException("Flat classpath doesn't support the classpath invalidation")
-
-    implicit object ClassPathOrdering extends Ordering[PlatformClassPath] {
-      def compare(a:PlatformClassPath, b:PlatformClassPath) = a.asClassPathString compare b.asClassPathString
+    implicit object ClassPathOrdering extends Ordering[ClassFileLookup[AbstractFile]] {
+      def compare(a:ClassFileLookup[AbstractFile], b:ClassFileLookup[AbstractFile]) = a.asClassPathString compare b.asClassPathString
     }
     val invalidated, failed = new mutable.ListBuffer[ClassSymbol]
-    classPath match {
-      case cp: MergedClassPath[_] =>
-        def assoc(path: String): List[(PlatformClassPath, PlatformClassPath)] = {
-          val dir = AbstractFile.getDirectory(path)
-          val canonical = dir.canonicalPath
-          def matchesCanonical(e: ClassPath[_]) = e.origin match {
-            case Some(opath) =>
-              AbstractFile.getDirectory(opath).canonicalPath == canonical
-            case None =>
-              false
-          }
-          cp.entries find matchesCanonical match {
-            case Some(oldEntry) =>
-              List(oldEntry -> cp.context.newClassPath(dir))
-            case None =>
-              error(s"Error adding entry to classpath. During invalidation, no entry named $path in classpath $classPath")
-              List()
-          }
-        }
-        val subst = immutable.TreeMap(paths flatMap assoc: _*)
-        if (subst.nonEmpty) {
-          platform updateClassPath subst
-          informProgress(s"classpath updated on entries [${subst.keys mkString ","}]")
-          def mkClassPath(elems: Iterable[PlatformClassPath]): PlatformClassPath =
-            if (elems.size == 1) elems.head
-            else new MergedClassPath(elems, recursiveClassPath.context)
-          val oldEntries = mkClassPath(subst.keys)
-          val newEntries = mkClassPath(subst.values)
-          mergeNewEntries(newEntries, RootClass, Some(recursiveClassPath), Some(oldEntries), invalidated, failed)
-        }
+
+    def assoc(path: String): Option[(ClassFileLookup[AbstractFile], ClassFileLookup[AbstractFile])] = {
+      def origin(lookup: ClassFileLookup[AbstractFile]): Option[String] = lookup match {
+        case cp: ClassPath[_] => cp.origin
+        case cp: JFileDirectoryLookup[_] => Some(cp.dir.getPath)
+        case cp: ZipArchiveFileLookup[_] => Some(cp.zipFile.getPath)
+        case _ => None
+      }
+
+      def entries(lookup: ClassFileLookup[AbstractFile]): Seq[ClassFileLookup[AbstractFile]] = lookup match {
+        case cp: ClassPath[_] => cp.entries
+        case cp: AggregateFlatClassPath => cp.aggregates
+        case cp: FlatClassPath => Seq(cp)
+      }
+
+      val dir = AbstractFile.getDirectory(path) // if path is a `jar`, this is a FileZipArchive (isDirectory is true)
+      val canonical = dir.canonicalPath         // this is the canonical path of the .jar
+      def matchesCanonical(e: ClassFileLookup[AbstractFile]) = origin(e) match {
+        case Some(opath) =>
+          AbstractFile.getDirectory(opath).canonicalPath == canonical
+        case None =>
+          false
+      }
+      entries(classPath) find matchesCanonical match {
+        case Some(oldEntry) =>
+          Some(oldEntry -> ClassFileLookup.createForFile(dir, classPath, settings))
+        case None =>
+          error(s"Error adding entry to classpath. During invalidation, no entry named $path in classpath $classPath")
+          None
+      }
+    }
+    val subst = immutable.TreeMap(paths flatMap assoc: _*)
+    if (subst.nonEmpty) {
+      platform updateClassPath subst
+      informProgress(s"classpath updated on entries [${subst.keys mkString ","}]")
+      def mkClassPath(elems: Iterable[ClassFileLookup[AbstractFile]]): ClassFileLookup[AbstractFile] =
+        if (elems.size == 1) elems.head
+        else ClassFileLookup.createAggregate(elems, classPath)
+      val oldEntries = mkClassPath(subst.keys)
+      val newEntries = mkClassPath(subst.values)
+      classPath match {
+        case rcp: ClassPath[_] => mergeNewEntriesRecursive(
+          newEntries.asInstanceOf[ClassPath[AbstractFile]], RootClass, Some(rcp), Some(oldEntries.asInstanceOf[ClassPath[AbstractFile]]),
+          invalidated, failed)
+
+        case fcp: FlatClassPath => mergeNewEntriesFlat(
+          RootClass, "",
+          oldEntries.asInstanceOf[FlatClassPath], newEntries.asInstanceOf[FlatClassPath], fcp,
+          invalidated, failed)
+      }
     }
     def show(msg: String, syms: scala.collection.Traversable[Symbol]) =
       if (syms.nonEmpty)
@@ -876,22 +894,22 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
    *
    *  Here, old means classpath, and sym means symboltable. + is presence of an entry in its column, - is absence.
    */
-  private def mergeNewEntries(newEntries: PlatformClassPath, root: ClassSymbol,
-             allEntries: OptClassPath, oldEntries: OptClassPath,
+  private def mergeNewEntriesRecursive(newEntries: ClassPath[AbstractFile], root: ClassSymbol,
+             allEntries: Option[ClassPath[AbstractFile]], oldEntries: Option[ClassPath[AbstractFile]],
              invalidated: mutable.ListBuffer[ClassSymbol], failed: mutable.ListBuffer[ClassSymbol]) {
     ifDebug(informProgress(s"syncing $root, $oldEntries -> $newEntries"))
 
-    val getName: ClassPath[AbstractFile] => String = (_.name)
-    def hasClasses(cp: OptClassPath) = cp.isDefined && cp.get.classes.nonEmpty
+    val getPackageName: ClassPath[AbstractFile] => String = _.name
+    def hasClasses(cp: Option[ClassPath[AbstractFile]]) = cp.isDefined && cp.get.classes.nonEmpty
     def invalidateOrRemove(root: ClassSymbol) = {
       allEntries match {
         case Some(cp) => root setInfo new loaders.PackageLoader(cp)
         case None => root.owner.info.decls unlink root.sourceModule
       }
       invalidated += root
     }
-    def subPackage(cp: PlatformClassPath, name: String): OptClassPath =
-      cp.packages find (cp1 => getName(cp1) == name)
+    def subPackage(cp: ClassPath[AbstractFile], name: String): Option[ClassPath[AbstractFile]] =
+      cp.packages find (cp1 => getPackageName(cp1) == name)
 
     val classesFound = hasClasses(oldEntries) || newEntries.classes.nonEmpty
     if (classesFound && !isSystemPackageClass(root)) {
@@ -901,22 +919,81 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
         if (root.isRoot) invalidateOrRemove(EmptyPackageClass)
         else failed += root
       }
-      if (!oldEntries.isDefined) invalidateOrRemove(root)
+      if (oldEntries.isEmpty) invalidateOrRemove(root)
       else
-        for (pstr <- newEntries.packages.map(getName)) {
+        for (pstr <- newEntries.packages.map(getPackageName)) {
           val pname = newTermName(pstr)
           val pkg = (root.info decl pname) orElse {
             // package does not exist in symbol table, create symbol to track it
-            assert(!subPackage(oldEntries.get, pstr).isDefined)
+            assert(subPackage(oldEntries.get, pstr).isEmpty)
             loaders.enterPackage(root, pstr, new loaders.PackageLoader(allEntries.get))
           }
-          mergeNewEntries(subPackage(newEntries, pstr).get, pkg.moduleClass.asClass,
+          mergeNewEntriesRecursive(subPackage(newEntries, pstr).get, pkg.moduleClass.asClass,
                           subPackage(allEntries.get, pstr), subPackage(oldEntries.get, pstr),
                           invalidated, failed)
         }
     }
   }
 
+  /**
+   * Merges new classpath entries into the symbol table
+   *
+   * @param packageClass    The ClassSymbol for the package being updated
+   * @param fullPackageName The full name of the package being updated
+   * @param oldEntries      The classpath that was removed, it is no longer part of fullClasspath
+   * @param newEntries      The classpath that was added, it is already part of fullClasspath
+   * @param fullClasspath   The full classpath, equivalent to global.classPath
+   * @param invalidated     A ListBuffer collecting the invalidated package classes
+   * @param failed          A ListBuffer collecting system package classes which could not be invalidated
+   *
+   * If either oldEntries or newEntries contains classes in the current package, the package symbol
+   * is re-initialized to a fresh package loader, provided that a corresponding package exists in
+   * fullClasspath. Otherwise it is removed.
+   *
+   * Otherwise, sub-packages in newEntries are looked up in the symbol table (created if
+   * non-existent) and the merge function is called recursively.
+   */
+  private def mergeNewEntriesFlat(
+      packageClass: ClassSymbol, fullPackageName: String,
+      oldEntries: FlatClassPath, newEntries: FlatClassPath, fullClasspath: FlatClassPath,
+      invalidated: mutable.ListBuffer[ClassSymbol], failed: mutable.ListBuffer[ClassSymbol]): Unit = {
+    ifDebug(informProgress(s"syncing $packageClass, $oldEntries -> $newEntries"))
+
+    def packageExists(cp: FlatClassPath): Boolean = {
+      val (parent, _) = PackageNameUtils.separatePkgAndClassNames(fullPackageName)
+      cp.packages(parent).exists(_.name == fullPackageName)
+    }
+
+    def invalidateOrRemove(pkg: ClassSymbol) = {
+      if (packageExists(fullClasspath))
+        pkg setInfo new loaders.PackageLoaderUsingFlatClassPath(fullPackageName, fullClasspath)
+      else
+        pkg.owner.info.decls unlink pkg.sourceModule
+      invalidated += pkg
+    }
+
+    val classesFound = oldEntries.classes(fullPackageName).nonEmpty || newEntries.classes(fullPackageName).nonEmpty
+    if (classesFound) {
+      // if the package contains classes either in oldEntries or newEntries, the package is invalidated (or removed if there are no more classes in it)
+      if (!isSystemPackageClass(packageClass)) invalidateOrRemove(packageClass)
+      else if (packageClass.isRoot) invalidateOrRemove(EmptyPackageClass)
+      else failed += packageClass
+    } else {
+      // no new or removed classes in the current package
+      for (p <- newEntries.packages(fullPackageName)) {
+        val (_, subPackageName) = PackageNameUtils.separatePkgAndClassNames(p.name)
+        val subPackage = packageClass.info.decl(newTermName(subPackageName)) orElse {
+          // package does not exist in symbol table, create a new symbol
+          loaders.enterPackage(packageClass, subPackageName, new loaders.PackageLoaderUsingFlatClassPath(p.name, fullClasspath))
+        }
+        mergeNewEntriesFlat(
+          subPackage.moduleClass.asClass, p.name,
+          oldEntries, newEntries, fullClasspath,
+          invalidated, failed)
+      }
+    }
+  }
+
   // ----------- Runs ---------------------------------------
 
   private var curRun: Run = null

src/compiler/scala/tools/nsc/backend/JavaPlatform.scala

@@ -7,9 +7,9 @@ package scala.tools.nsc
 package backend
 
 import io.AbstractFile
-import scala.tools.nsc.classpath.FlatClassPath
+import scala.tools.nsc.classpath.{AggregateFlatClassPath, FlatClassPath}
 import scala.tools.nsc.settings.ClassPathRepresentationType
-import scala.tools.nsc.util.{ ClassPath, DeltaClassPath, MergedClassPath }
+import scala.tools.nsc.util.{ClassFileLookup, ClassPath, MergedClassPath}
 import scala.tools.util.FlatClassPathResolver
 import scala.tools.util.PathResolver
 
@@ -29,16 +29,29 @@ trait JavaPlatform extends Platform {
     currentClassPath.get
   }
 
-  private[nsc] lazy val flatClassPath: FlatClassPath = {
+  private[nsc] var currentFlatClassPath: Option[FlatClassPath] = None
+
+  private[nsc] def flatClassPath: FlatClassPath = {
     assert(settings.YclasspathImpl.value == ClassPathRepresentationType.Flat,
       "To use flat classpath representation you must enable it with -YclasspathImpl:flat compiler option.")
 
-    new FlatClassPathResolver(settings).result
+    if (currentFlatClassPath.isEmpty) currentFlatClassPath = Some(new FlatClassPathResolver(settings).result)
+    currentFlatClassPath.get
   }
 
   /** Update classpath with a substituted subentry */
-  def updateClassPath(subst: Map[ClassPath[AbstractFile], ClassPath[AbstractFile]]) =
-    currentClassPath = Some(new DeltaClassPath(currentClassPath.get, subst))
+  def updateClassPath(subst: Map[ClassFileLookup[AbstractFile], ClassFileLookup[AbstractFile]]) = global.classPath match {
+    case cp: ClassPath[AbstractFile] =>
+      val s = subst.asInstanceOf[Map[ClassPath[AbstractFile], ClassPath[AbstractFile]]]
+      currentClassPath = Some(new MergedClassPath(cp.entries map (e => s.getOrElse(e, e)), cp.context))
+
+    case AggregateFlatClassPath(entries) =>
+      val s = subst.asInstanceOf[Map[FlatClassPath, FlatClassPath]]
+      currentFlatClassPath = Some(AggregateFlatClassPath(entries map (e => s.getOrElse(e, e))))
+
+    case cp: FlatClassPath =>
+      currentFlatClassPath = Some(subst.getOrElse(cp, cp).asInstanceOf[FlatClassPath])
+  }
 
   def platformPhases = List(
     flatten,  // get rid of inner classes

src/compiler/scala/tools/nsc/backend/Platform.scala

@@ -6,7 +6,7 @@
 package scala.tools.nsc
 package backend
 
-import util.ClassPath
+import util.{ClassFileLookup, ClassPath}
 import io.AbstractFile
 import scala.tools.nsc.classpath.FlatClassPath
 
@@ -23,7 +23,7 @@ trait Platform {
   private[nsc] def flatClassPath: FlatClassPath
 
   /** Update classpath with a substitution that maps entries to entries */
-  def updateClassPath(subst: Map[ClassPath[AbstractFile], ClassPath[AbstractFile]])
+  def updateClassPath(subst: Map[ClassFileLookup[AbstractFile], ClassFileLookup[AbstractFile]])
 
   /** Any platform-specific phases. */
   def platformPhases: List[SubComponent]

src/compiler/scala/tools/nsc/classpath/AggregateFlatClassPath.scala

@@ -125,3 +125,15 @@ case class AggregateFlatClassPath(aggregates: Seq[FlatClassPath]) extends FlatCl
   private def classesGetter(pkg: String) = (cp: FlatClassPath) => cp.classes(pkg)
   private def sourcesGetter(pkg: String) = (cp: FlatClassPath) => cp.sources(pkg)
 }
+
+object AggregateFlatClassPath {
+  def createAggregate(parts: FlatClassPath*): FlatClassPath = {
+    val elems = new ArrayBuffer[FlatClassPath]()
+    parts foreach {
+      case AggregateFlatClassPath(ps) => elems ++= ps
+      case p => elems += p
+    }
+    if (elems.size == 1) elems.head
+    else AggregateFlatClassPath(elems.toIndexedSeq)
+  }
+}

src/compiler/scala/tools/nsc/util/ClassFileLookup.scala

@@ -3,6 +3,8 @@
  */
 package scala.tools.nsc.util
 
+import scala.tools.nsc.Settings
+import scala.tools.nsc.classpath.{AggregateFlatClassPath, FlatClassPath, FlatClassPathFactory}
 import scala.tools.nsc.io.AbstractFile
 import java.net.URL
 
@@ -39,6 +41,25 @@ trait ClassFileLookup[T] {
   def asSourcePathString: String
 }
 
+object ClassFileLookup {
+  def createForFile(f: AbstractFile, current: ClassFileLookup[AbstractFile], settings: Settings): ClassFileLookup[AbstractFile] = current match {
+    case cp: ClassPath[_] => cp.context.newClassPath(f)
+    case _: FlatClassPath => FlatClassPathFactory.newClassPath(f, settings)
+  }
+
+  def createAggregate(elems: Iterable[ClassFileLookup[AbstractFile]], current: ClassFileLookup[AbstractFile]): ClassFileLookup[AbstractFile] = {
+    assert(elems.nonEmpty)
+    if (elems.size == 1) elems.head
+    else current match {
+      case cp: ClassPath[_] =>
+        new MergedClassPath(elems.asInstanceOf[Iterable[ClassPath[AbstractFile]]], cp.context)
+
+      case _: FlatClassPath =>
+        AggregateFlatClassPath.createAggregate(elems.asInstanceOf[Iterable[FlatClassPath]].toSeq : _*)
+    }
+  }
+}
+
 /**
  * Represents classes which can be loaded with a ClassfileLoader and/or SourcefileLoader.
  */