Implement early output and early analysis

For modular pipelining we need both early output (JAR file containing Scala sig files) and early Analysis (Zinc internal information). This adds `IncOptions#earlyOutput` and `Lookup#storeEarlyAnalysis` so the early artifacts can be generated during compile phases.
sbt · Jul 6, 2020 · 1b7081d · 1b7081d
1 parent 602cfe5
commit 1b7081d
Show file tree

Hide file tree

Showing 27 changed files with 722 additions and 149 deletions.
diff --git a/internal/compiler-bridge/src/main/scala/xsbt/API.scala b/internal/compiler-bridge/src/main/scala/xsbt/API.scala
@@ -34,6 +34,10 @@ final class API(val global: CallbackGlobal) extends Compat with GlobalHelpers wi
       val start = System.currentTimeMillis
       super.run()
 
+      //  We're running right after pickling, so store pickles now.
+      val pickleData = Compat.picklePaths(currentRun)
+      callback.pickleData(pickleData.toArray)
+
       // After processing all units, register generated classes
       registerGeneratedClasses(nonLocalClassSymbolsInCurrentUnits.iterator)
       nonLocalClassSymbolsInCurrentUnits.clear()

diff --git a/internal/compiler-bridge/src/main/scala_2.10/xsbt/Compat.scala b/internal/compiler-bridge/src/main/scala_2.10/xsbt/Compat.scala
@@ -13,6 +13,7 @@ package xsbt
 
 import java.io.PrintWriter
 import java.nio.file.Path
+import xsbti.PickleData
 import xsbti.compile.Output
 import scala.reflect.{ internal => sri }
 import scala.reflect.internal.{ util => sriu }
@@ -190,6 +191,9 @@ object Compat {
   }
 
   def plainNioFile(path: Path): AbstractFile = new PlainNioFile(path)
+
+  // No pileline pickling in 2.10
+  def picklePaths(run: Global#Run) = Iterable.empty[PickleData]
 }
 
 private trait CachedCompilerCompat { self: CachedCompiler0 =>

diff --git a/internal/compiler-bridge/src/main/scala_2.11-12/xsbt/Compat.scala b/internal/compiler-bridge/src/main/scala_2.11-12/xsbt/Compat.scala
@@ -12,10 +12,11 @@
 package xsbt
 
 import java.io.PrintWriter
-import java.nio.file.Path
+import java.nio.file.{ Path, Paths }
+import xsbti.PickleData
 import xsbti.compile.Output
-
-import scala.tools.nsc.Settings
+import scala.collection.mutable
+import scala.tools.nsc.{ Global, Settings }
 import scala.reflect.io.AbstractFile
 
 abstract class Compat
@@ -29,6 +30,37 @@ object Compat {
   def replReporter(settings: Settings, writer: PrintWriter) = writer
 
   def plainNioFile(path: Path): AbstractFile = new PlainNioFile(path)
+
+  // Prepare pickle data for eventual storage, computing path within jar file from symbol ownership
+  // and storing data in a class that does not rely on a shared scala library.
+  // This is almost verbatim copied from scala.tools.nsc.PipelineMain, except that actually writing to the jar file
+  // is deferred to AnalysisCallback, after the final incremental compilation cycle.
+  def picklePaths[G <: Global](run: G#Run): Iterable[PickleData] = {
+    val rootPath = Paths.get("__ROOT__")
+    val dirs = mutable.Map[G#Symbol, Path]()
+    def packageDir(packSymbol: G#Symbol): Path = {
+      if (packSymbol.isEmptyPackageClass) rootPath
+      else if (dirs.contains(packSymbol)) dirs(packSymbol)
+      else if (packSymbol.owner.isRoot) {
+        val subDir = rootPath.resolve(packSymbol.encodedName)
+        dirs.put(packSymbol, subDir)
+        subDir
+      } else {
+        val base = packageDir(packSymbol.owner)
+        val subDir = base.resolve(packSymbol.encodedName)
+        dirs.put(packSymbol, subDir)
+        subDir
+      }
+    }
+
+    for { (s, p) <- run.symData } yield {
+      val base = packageDir(s.owner)
+      val path = base.resolve(s.encodedName + ".sig")
+      //        val path = symToPath(s,true)
+      val fqcn = s.fullNameString
+      PickleData.of(p, fqcn, p.bytes, p.writeIndex, path)
+    }
+  }
 }
 
 /** Defines compatibility utils for [[ZincCompiler]]. */

diff --git a/internal/compiler-bridge/src/main/scala_2.13/xsbt/Compat.scala b/internal/compiler-bridge/src/main/scala_2.13/xsbt/Compat.scala
@@ -12,11 +12,13 @@
 package xsbt
 
 import java.io.PrintWriter
-import java.nio.file.Path
+import java.nio.file.{ Path, Paths }
+import xsbti.PickleData
 import xsbti.compile.Output
-import scala.tools.nsc.Settings
+import scala.tools.nsc.{ Global, Settings }
 import scala.tools.nsc.interpreter.shell.ReplReporterImpl
 import scala.reflect.io.AbstractFile
+import scala.collection.mutable
 
 abstract class Compat
 object Compat {
@@ -30,6 +32,37 @@ object Compat {
     new ReplReporterImpl(settings, writer)
 
   def plainNioFile(path: Path): AbstractFile = new PlainNioFile(path)
+
+  // Prepare pickle data for eventual storage, computing path within jar file from symbol ownership
+  // and storing data in a class that does not rely on a shared scala library.
+  // This is almost verbatim copied from scala.tools.nsc.PipelineMain, except that actually writing to the jar file
+  // is deferred to AnalysisCallback, after the final incremental compilation cycle.
+  def picklePaths[G <: Global](run: G#Run): Iterable[PickleData] = {
+    val rootPath = Paths.get("__ROOT__")
+    val dirs = mutable.Map[G#Symbol, Path]()
+    def packageDir(packSymbol: G#Symbol): Path = {
+      if (packSymbol.isEmptyPackageClass) rootPath
+      else if (dirs.contains(packSymbol)) dirs(packSymbol)
+      else if (packSymbol.owner.isRoot) {
+        val subDir = rootPath.resolve(packSymbol.encodedName)
+        dirs.put(packSymbol, subDir)
+        subDir
+      } else {
+        val base = packageDir(packSymbol.owner)
+        val subDir = base.resolve(packSymbol.encodedName)
+        dirs.put(packSymbol, subDir)
+        subDir
+      }
+    }
+
+    for { (s, p) <- run.symData } yield {
+      val base = packageDir(s.owner)
+      val path = base.resolve(s.encodedName + ".sig")
+      //        val path = symToPath(s,true)
+      val fqcn = s.fullNameString
+      PickleData.of(p, fqcn, p.bytes, p.writeIndex, path)
+    }
+  }
 }
 
 /** Defines compatibility utils for [[ZincCompiler]]. */

diff --git a/internal/compiler-interface/src/main/contraband-java/xsbti/PickleData.java b/internal/compiler-interface/src/main/contraband-java/xsbti/PickleData.java
@@ -0,0 +1,76 @@
+/**
+ * This code is generated using [[https://www.scala-sbt.org/contraband/ sbt-contraband]].
+ */
+
+// DO NOT EDIT MANUALLY
+package xsbti;
+/** A wrapper around PickleBuffer https://github.com/scala/scala/blob/v2.13.1/src/reflect/scala/reflect/internal/pickling/PickleBuffer.scala */
+public final class PickleData implements java.io.Serializable {
+
+    public static PickleData create(Object _underlying, String _fqcn, byte[] _data, int _writeIndex, java.nio.file.Path _path) {
+        return new PickleData(_underlying, _fqcn, _data, _writeIndex, _path);
+    }
+    public static PickleData of(Object _underlying, String _fqcn, byte[] _data, int _writeIndex, java.nio.file.Path _path) {
+        return new PickleData(_underlying, _fqcn, _data, _writeIndex, _path);
+    }
+    private Object underlying;
+    private String fqcn;
+    private byte[] data;
+    private int writeIndex;
+    private java.nio.file.Path path;
+    protected PickleData(Object _underlying, String _fqcn, byte[] _data, int _writeIndex, java.nio.file.Path _path) {
+        super();
+        underlying = _underlying;
+        fqcn = _fqcn;
+        data = _data;
+        writeIndex = _writeIndex;
+        path = _path;
+    }
+
+    public Object underlying() {
+        return this.underlying;
+    }
+    public String fqcn() {
+        return this.fqcn;
+    }
+    public byte[] data() {
+        return this.data;
+    }
+    public int writeIndex() {
+        return this.writeIndex;
+    }
+    public java.nio.file.Path path() {
+        return this.path;
+    }
+    public PickleData withUnderlying(Object underlying) {
+        return new PickleData(underlying, fqcn, data, writeIndex, path);
+    }
+    public PickleData withFqcn(String fqcn) {
+        return new PickleData(underlying, fqcn, data, writeIndex, path);
+    }
+    public PickleData withData(byte[] data) {
+        return new PickleData(underlying, fqcn, data, writeIndex, path);
+    }
+    public PickleData withWriteIndex(int writeIndex) {
+        return new PickleData(underlying, fqcn, data, writeIndex, path);
+    }
+    public PickleData withPath(java.nio.file.Path path) {
+        return new PickleData(underlying, fqcn, data, writeIndex, path);
+    }
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        } else if (!(obj instanceof PickleData)) {
+            return false;
+        } else {
+            PickleData o = (PickleData)obj;
+            return this.underlying().equals(o.underlying()) && this.fqcn().equals(o.fqcn()) && java.util.Arrays.equals(this.data(), o.data()) && (this.writeIndex() == o.writeIndex()) && this.path().equals(o.path());
+        }
+    }
+    public int hashCode() {
+        return 37 * (37 * (37 * (37 * (37 * (37 * (17 + "xsbti.PickleData".hashCode()) + underlying().hashCode()) + fqcn().hashCode()) + java.util.Arrays.hashCode(data())) + Integer.valueOf(writeIndex()).hashCode()) + path().hashCode());
+    }
+    public String toString() {
+        return "PickleData("  + "underlying: " + underlying() + ", " + "fqcn: " + fqcn() + ", " + "data: " + data() + ", " + "writeIndex: " + writeIndex() + ", " + "path: " + path() + ")";
+    }
+}