Merge pull request #351 from scalacenter/protobuf

Add protobuf-based analysis binary store

build.sbt

@@ -197,13 +197,15 @@ lazy val zincCompile = (project in file("zinc-compile"))
   )
   .configure(addSbtUtilTracking)
 
-// Persists the incremental data structures using SBinary
+// Persists the incremental data structures using Protobuf
 lazy val zincPersist = (project in internalPath / "zinc-persist")
   .dependsOn(zincCore, zincCore % "test->test")
   .configure(addBaseSettingsAndTestDeps)
   .settings(
     name := "zinc Persist",
-    libraryDependencies += sbinary
+    libraryDependencies += sbinary,
+    compileOrder := sbt.CompileOrder.Mixed,
+    PB.targets in Compile := List(scalapb.gen() -> (sourceManaged in Compile).value)
   )
 
 // Implements the core functionality of detecting and propagating changes incrementally.

internal/compiler-interface/src/main/java/xsbti/api/Modifiers.java

@@ -37,6 +37,25 @@ public Modifiers(boolean isAbstract, boolean isOverride, boolean isFinal, boolea
 		);
 	}
 
+	/**
+     * Allow to set the modifiers from a flags byte where:
+	 *
+	 *   1. The first bit tells if has an abstract modifier.
+	 *   2. The second bit tells if has an override modifier.
+	 *   3. The third bit tells if has an final modifier.
+	 *   4. The fourth bit tells if has an sealed modifier.
+	 *   5. The fifth bit tells if has an implicit modifier.
+	 *   6. The sixth bit tells if has an lazy modifier.
+	 *   7. The seventh bit tells if has an macro modifier.
+	 *   8. The eighth bit tells if has an super accessor modifier.
+	 *
+	 * This method is not part of the public API and it may be removed at any point.
+	 * @param flags An instance of byte encoding the modifiers.
+	 */
+	protected Modifiers(byte flags) {
+		this.flags = flags;
+	}
+
 	private final byte flags;
 
 	private boolean flag(int bit)

internal/zinc-compile-core/src/main/java/xsbti/compile/CompilerCache.java

@@ -1,3 +1,10 @@
+/*
+ * Zinc - The incremental compiler for Scala.
+ * Copyright 2011 - 2017, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * This software is released under the terms written in LICENSE.
+ */
+
 package xsbti.compile;
 
 /**

internal/zinc-core/src/main/scala/sbt/internal/inc/OutputGroup.scala → internal/zinc-core/src/main/scala/sbt/internal/inc/SimpleOutputGroup.scala

@@ -9,7 +9,7 @@ package sbt.internal.inc
 
 import java.io.File
 
-import xsbti.compile.OutputGroup
-
 case class SimpleOutputGroup(getSourceDirectory: File, getOutputDirectory: File)
-    extends OutputGroup
+    extends xsbti.compile.OutputGroup {
+  override def toString = s"OutputGroup($getSourceDirectory -> $getOutputDirectory)"
+}

internal/zinc-core/src/main/scala/sbt/internal/inc/Stamp.scala

@@ -16,6 +16,7 @@ import java.util.Optional
 import sbt.io.{ Hash => IOHash }
 import xsbti.compile.analysis.{ ReadStamps, Stamp }
 
+import scala.collection.immutable.TreeMap
 import scala.util.matching.Regex
 
 /**
@@ -58,10 +59,10 @@ trait WithPattern { protected def Pattern: Regex }
 import java.lang.{ Long => BoxedLong }
 
 /** Define the hash of the file contents. It's a typical stamp for compilation sources. */
-final class Hash(val value: Array[Byte]) extends StampBase {
-  val hexHash = IOHash.toHex(value)
+final class Hash private[inc] (val hexHash: String) extends StampBase {
+  // Assumes `hexHash` is a hexadecimal value.
   override def writeStamp: String = s"hash($hexHash)"
-  override def getValueId: Int = java.util.Arrays.hashCode(value)
+  override def getValueId: Int = hexHash.hashCode()
   override def getHash: Optional[String] = Optional.of(hexHash)
   override def getLastModified: Optional[BoxedLong] = Optional.empty[BoxedLong]
 }
@@ -96,19 +97,21 @@ object Stamp {
   private final val maxModificationDifferenceInMillis = 100L
   implicit val equivStamp: Equiv[Stamp] = new Equiv[Stamp] {
     def equiv(a: Stamp, b: Stamp) = (a, b) match {
-      case (h1: Hash, h2: Hash) => h1.value sameElements h2.value
+      case (h1: Hash, h2: Hash) => h1.hexHash == h2.hexHash
       // Windows is handling this differently sometimes...
       case (lm1: LastModified, lm2: LastModified) =>
         lm1.value == lm2.value ||
           Math.abs(lm1.value - lm2.value) < maxModificationDifferenceInMillis
-      case (EmptyStamp, EmptyStamp) => true
-      case _                        => false
+      case (stampA, stampB) =>
+        // This part of code should not depend on `equals`
+        // Checking for (EmptyStamp, EmptyStamp) produces SOE
+        stampA.eq(EmptyStamp) && stampB.eq(EmptyStamp)
     }
   }
 
   def fromString(s: String): Stamp = s match {
     case EmptyStamp.Value            => EmptyStamp
-    case Hash.Pattern(value)         => new Hash(IOHash.fromHex(value))
+    case Hash.Pattern(value)         => new Hash(value)
     case LastModified.Pattern(value) => new LastModified(java.lang.Long.parseLong(value))
     case _ =>
       throw new IllegalArgumentException("Unrecognized Stamp string representation: " + s)
@@ -123,7 +126,7 @@ object Stamper {
     catch { case i: IOException => EmptyStamp }
   }
 
-  val forHash = (toStamp: File) => tryStamp(new Hash(IOHash(toStamp)))
+  val forHash = (toStamp: File) => tryStamp(new Hash(IOHash.toHex(IOHash(toStamp))))
   val forLastModified = (toStamp: File) => tryStamp(new LastModified(toStamp.lastModified()))
 }
 
@@ -141,7 +144,8 @@ object Stamps {
     new InitialStamps(prodStamp, srcStamp, binStamp)
 
   def empty: Stamps = {
-    val eSt = Map.empty[File, Stamp]
+    // Use a TreeMap to avoid sorting when serializing
+    val eSt = TreeMap.empty[File, Stamp]
     apply(eSt, eSt, eSt)
   }
   def apply(products: Map[File, Stamp],

internal/zinc-persist/src/main/java/xsbti/compile/analysis/GenericMapper.java

@@ -0,0 +1,93 @@
+/*
+ * Zinc - The incremental compiler for Scala.
+ * Copyright 2011 - 2017, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * This software is released under the terms written in LICENSE.
+ */
+
+package xsbti.compile.analysis;
+
+import xsbti.compile.MiniSetup;
+
+import java.io.File;
+
+
+/**
+ * Defines a generic interface to map the values of the analysis file 1-to-1.
+ */
+public interface GenericMapper {
+    /**
+     * @param sourceFile A source file to be compiled.
+     * @return A modified source file.
+     */
+    File mapSourceFile(File sourceFile);
+
+    /**
+     * @param binaryFile A binary dependency of the sources to be compiled.
+     * @return A modified binary file.
+     */
+    File mapBinaryFile(File binaryFile);
+
+    /**
+     * @param productFile A product file (class file) produced by the compiler.
+     * @return A modified product file.
+     */
+    File mapProductFile(File productFile);
+
+    /**
+     * @param outputDir The output dir where the compiler will output the products.
+     * @return A modified output dir.
+     */
+    File mapOutputDir(File outputDir);
+
+    /**
+     * @param sourceDir The source dir where the compiler will look for the sources.
+     * @return A modified source dir.
+     */
+    File mapSourceDir(File sourceDir);
+
+    /**
+     * @param classpathEntry The classpath entry to be passed to the compiler.
+     * @return A modified classpath entry.
+     */
+    File mapClasspathEntry(File classpathEntry);
+
+    /**
+     * @param javacOption An option to be passed to the Java compiler.
+     * @return A compiler option.
+     */
+    String mapJavacOption(String javacOption);
+
+    /**
+     * @param scalacOption An options to be passed to the Scala compiler.
+     * @return A compiler option.
+     */
+    String mapScalacOption(String scalacOption);
+
+    /**
+     * @param file The owner of the stamp.
+     * @param binaryStamp A stamp associated to a binary file.
+     * @return A transformed stamp.
+     */
+    Stamp mapBinaryStamp(File file, Stamp binaryStamp);
+
+    /**
+     * @param file The owner of the stamp.
+     * @param sourceStamp A stamp associated to a source file.
+     * @return A transformed stamp.
+     */
+    Stamp mapSourceStamp(File file, Stamp sourceStamp);
+
+    /**
+     * @param file The owner of the stamp.
+     * @param productStamp A stamp associated to a product file.
+     * @return A transformed stamp.
+     */
+    Stamp mapProductStamp(File file, Stamp productStamp);
+
+    /**
+     * @param miniSetup The simple compile setup that is serialized in the analysis file.
+     * @return A transformed mini setup.
+     */
+    MiniSetup mapMiniSetup(MiniSetup miniSetup);
+}

internal/zinc-persist/src/main/java/xsbti/compile/analysis/ReadMapper.java

@@ -0,0 +1,149 @@
+/*
+ * Zinc - The incremental compiler for Scala.
+ * Copyright 2011 - 2017, Lightbend, Inc.
+ * Copyright 2008 - 2010, Mark Harrah
+ * This software is released under the terms written in LICENSE.
+ */
+
+package xsbti.compile.analysis;
+
+import sbt.internal.inc.mappers.NaiveRelativeReadMapper;
+import sbt.internal.inc.mappers.RelativeReadMapper;
+import xsbti.compile.MiniSetup;
+
+import java.io.File;
+import java.nio.file.Path;
+
+/**
+ * Defines a reader-only mapper interface that is used by Zinc after reading
+ * the analysis from an instance of an {@link xsbti.compile.AnalysisStore}.
+ *
+ * This interface is useful to make the analysis file machine-independent and
+ * allow third parties to distribute them around.
+ */
+public interface ReadMapper extends GenericMapper {
+
+    /**
+     * Defines a mapper that reads from a machine-independent analysis file.
+     *
+     * An analysis file is machine independent if all the paths are relative to a path
+     * and no information about the machine that produced it is stored -- only
+     * information about the structure of the project from a given build tool is persisted.
+     *
+     * The mapper makes sure that the analysis file looks like if it was generated by
+     * the local machine it's executed on.
+     *
+     * <b>Important note</b>: The assumption that all paths can be made relative to a concrete
+     * position is not always correct. There is no guarantee that the paths of the caches, the
+     * artifacts, the classpath entries, the compilation inputs and outputs, et cetera share
+     * the same prefix. If this is not the case this reader will fail at runtime.
+     *
+     * Such assumption is broken in our test infrastructure: `lib_managed` does not share
+     * the same prefix, and without redefining it, it fails. Note that this is a conscious
+     * design decision of the relative read and write mappers. They are focused on simplicity.
+     * Build tools that need more careful handling of paths should create their own read and
+     * write mappers.
+     *
+     *
+     * @param projectRootPath The path on which we want to "mount" all the relative paths in analysis.
+     * @return A read mapper to pass in to {@link sbt.internal.inc.FileAnalysisStore}.
+     */
+    static ReadMapper getMachineIndependentMapper(Path projectRootPath) {
+        return new NaiveRelativeReadMapper(projectRootPath);
+    }
+
+    /**
+     * Defines a mapper that reads from a machine-independent analysis file.
+     *
+     * An analysis file is machine independent if all the paths are relative to a path
+     * and no information about the machine that produced it is stored -- only
+     * information about the structure of the project from a given build tool is persisted.
+     *
+     * The mapper makes sure that the analysis file looks like if it was generated by
+     * the local machine it's executed on.
+     *
+     * This mapper will use the paths provided by {@link RootPaths} to make the file relative,
+     * allowing build tools to pass independent common root paths for different kind of files.
+     *
+     * In case users of this machine-independent mapper need further customisation, they
+     * can use the decorator pattern or roll their own.
+     *
+     * @param rootPaths The root paths to use internally by the mapper.
+     * @return A machine-independent read mapper to pass in to {@link sbt.internal.inc.FileAnalysisStore}.
+     */
+    static ReadMapper getMachineIndependentMapper(RootPaths rootPaths) {
+        return new RelativeReadMapper(rootPaths);
+    }
+
+    /**
+     * Defines an no-op read mapper.
+     *
+     * This is useful when users are not interested in distributing the analysis files
+     * and need to pass a read mapper to {@link sbt.internal.inc.FileAnalysisStore}.
+     *
+     * @return A no-op read mapper.
+     */
+    static ReadMapper getEmptyMapper() {
+        return new ReadMapper() {
+            @Override
+            public File mapSourceFile(File sourceFile) {
+                return sourceFile;
+            }
+
+            @Override
+            public File mapBinaryFile(File binaryFile) {
+                return binaryFile;
+            }
+
+            @Override
+            public File mapProductFile(File productFile) {
+                return productFile;
+            }
+
+            @Override
+            public File mapOutputDir(File outputDir) {
+                return outputDir;
+            }
+
+            @Override
+            public File mapSourceDir(File sourceDir) {
+                return sourceDir;
+            }
+
+            @Override
+            public File mapClasspathEntry(File classpathEntry) {
+                return classpathEntry;
+            }
+
+            @Override
+            public String mapJavacOption(String javacOption) {
+                return javacOption;
+            }
+
+            @Override
+            public String mapScalacOption(String scalacOption) {
+                return scalacOption;
+            }
+
+            @Override
+            public Stamp mapBinaryStamp(File file, Stamp binaryStamp) {
+                return binaryStamp;
+            }
+
+            @Override
+            public Stamp mapSourceStamp(File file, Stamp sourceStamp) {
+                return sourceStamp;
+            }
+
+            @Override
+            public Stamp mapProductStamp(File file, Stamp productStamp) {
+                return productStamp;
+            }
+
+            @Override
+            public MiniSetup mapMiniSetup(MiniSetup miniSetup) {
+                return miniSetup;
+            }
+        };
+    }
+}