Unify the compile graph among pipelined and normal compilation

Previously, we were using two different task scheduling implementations
for pipelined and normal compilation. Let's check what the overhead of
only using the pipelined task implementation is.

.scalafmt.conf

@@ -2,3 +2,6 @@ maxColumn = 100
 docstrings = JavaDoc
 assumeStandardLibraryStripMargin = true
 align.tokens=[]
+align.openParenCallSite = false
+align.openParenDefnSite = false
+binPack.literalArgumentLists = true

backend/src/main/scala/bloop/Compiler.scala

@@ -7,12 +7,12 @@ import java.io.File
 import java.net.URI
 
 import bloop.internal.Ecosystem
-import bloop.io.{AbsolutePath, Paths}
-import bloop.logging.Logger
+import bloop.io.AbsolutePath
 import bloop.reporter.Reporter
 import sbt.internal.inc.bloop.{BloopZincCompiler, CompileMode}
 import sbt.internal.inc.{FreshCompilerCache, Locate}
 import _root_.monix.eval.Task
+import bloop.util.CacheHashCode
 import sbt.internal.inc.bloop.internal.StopPipelining
 
 case class CompileInputs(
@@ -29,8 +29,7 @@ case class CompileInputs(
     classpathOptions: ClasspathOptions,
     previousResult: PreviousResult,
     reporter: Reporter,
-    mode: CompileMode,
-    logger: Logger
+    mode: CompileMode
 )
 
 object Compiler {
@@ -44,13 +43,22 @@ object Compiler {
 
   sealed trait Result
   object Result {
-    final case object Empty extends Result
-    final case class Blocked(on: List[String]) extends Result
-    final case class Cancelled(elapsed: Long) extends Result
-    final case class Failed(problems: List[xsbti.Problem], t: Option[Throwable], elapsed: Long)
-        extends Result
-    final case class Success(reporter: Reporter, previous: PreviousResult, elapsed: Long)
-        extends Result
+    final case object Empty extends Result with CacheHashCode
+    final case class Blocked(on: List[String]) extends Result with CacheHashCode
+    final case class Cancelled(elapsed: Long) extends Result with CacheHashCode
+    final case class GlobalError(problem: String) extends Result with CacheHashCode
+
+    final case class Success(
+        reporter: Reporter,
+        previous: PreviousResult,
+        elapsed: Long
+    ) extends Result with CacheHashCode
+
+    final case class Failed(
+        problems: List[xsbti.Problem],
+        t: Option[Throwable],
+        elapsed: Long
+    ) extends Result with CacheHashCode
 
     object Ok {
       def unapply(result: Result): Option[Result] = result match {
@@ -124,10 +132,11 @@ object Compiler {
     def elapsed: Long = ((System.nanoTime() - start).toDouble / 1e6).toLong
 
     import scala.util.{Success, Failure}
-    BloopZincCompiler.compile(inputs, compileInputs.mode, compileInputs.logger).materialize.map {
+    val logger = compileInputs.reporter.logger
+    BloopZincCompiler.compile(inputs, compileInputs.mode, logger).materialize.map {
       case Success(result) =>
-        val prev = PreviousResult.of(Optional.of(result.analysis()), Optional.of(result.setup()))
-        Result.Success(compileInputs.reporter, prev, elapsed)
+        val res = PreviousResult.of(Optional.of(result.analysis()), Optional.of(result.setup()))
+        Result.Success(compileInputs.reporter, res, elapsed)
       case Failure(f: StopPipelining) => Result.Blocked(f.failedProjectNames)
       case Failure(f: xsbti.CompileFailed) => Result.Failed(f.problems().toList, None, elapsed)
       case Failure(_: xsbti.CompileCancelled) => Result.Cancelled(elapsed)

backend/src/main/scala/bloop/util/CacheHashCode.scala

@@ -0,0 +1,7 @@
+package bloop.util
+
+trait CacheHashCode {
+  self: Product =>
+  // Cache hash code here so that `Dag.toDotGraph` doesn't recompute it all the time
+  override lazy val hashCode: Int = scala.util.hashing.MurmurHash3.productHash(self)
+}

frontend/src/main/scala/bloop/Project.scala

@@ -14,6 +14,7 @@ import config.{Config, ConfigEncoderDecoders}
 import config.Config.Platform
 import bloop.engine.ExecutionContext
 import bloop.engine.tasks.{ScalaJsToolchain, ScalaNativeToolchain}
+import bloop.util.CacheHashCode
 import ch.epfl.scala.{bsp => Bsp}
 
 final case class Project(
@@ -37,9 +38,8 @@ final case class Project(
     nativeToolchain: Option[ScalaNativeToolchain],
     sbt: Option[Config.Sbt],
     resolution: Option[Config.Resolution]
-) {
+) extends CacheHashCode {
   override def toString: String = s"$name"
-  override val hashCode: Int = scala.util.hashing.MurmurHash3.productHash(this)
 
   /** The bsp uri associated with this project. */
   val bspUri: Bsp.Uri = Bsp.Uri(ProjectUris.toUri(baseDirectory, name))

frontend/src/main/scala/bloop/bsp/BloopBspServices.scala

@@ -161,6 +161,7 @@ final class BloopBspServices(
               case Compiler.Result.Cancelled(_) => Nil
               case Compiler.Result.Blocked(_) => Nil
               case Compiler.Result.Success(_, _, _) => Nil
+              case Compiler.Result.GlobalError(problem) => List(problem)
               case Compiler.Result.Failed(problems, t, elapsed) =>
                 val acc = List(reportError(p, problems, elapsed))
                 t match {

frontend/src/main/scala/bloop/engine/Build.scala

@@ -3,16 +3,16 @@ package bloop.engine
 import bloop.Project
 import bloop.io.{AbsolutePath, FileTracker}
 import bloop.logging.Logger
+import bloop.util.CacheHashCode
 
 final case class Build private (
     origin: AbsolutePath,
     projects: List[Project],
     tracker: FileTracker
-) {
+) extends CacheHashCode {
 
   private val stringToProjects: Map[String, Project] = projects.map(p => p.name -> p).toMap
   private[bloop] val dags: List[Dag[Project]] = Dag.fromMap(stringToProjects)
-  override val hashCode: Int = scala.util.hashing.MurmurHash3.productHash(this)
 
   def getProjectFor(name: String): Option[Project] = stringToProjects.get(name)
   def getDagFor(project: Project): Dag[Project] =

frontend/src/main/scala/bloop/engine/Dag.scala

@@ -1,17 +1,14 @@
 package bloop.engine
 
 import bloop.Project
+import bloop.util.CacheHashCode
 import scalaz.Show
 
 sealed trait Dag[T]
 
-final case class Leaf[T](value: T) extends Dag[T] {
-  override val hashCode: Int = scala.util.hashing.MurmurHash3.productHash(this)
-}
+final case class Leaf[T](value: T) extends Dag[T] with CacheHashCode
 
-final case class Parent[T](value: T, children: List[Dag[T]]) extends Dag[T] {
-  override val hashCode: Int = scala.util.hashing.MurmurHash3.productHash(this)
-}
+final case class Parent[T](value: T, children: List[Dag[T]]) extends Dag[T] with CacheHashCode
 
 object Dag {
   class RecursiveCycle(path: List[Project])
@@ -142,6 +139,16 @@ object Dag {
     loop(dags.toSet, targets)
   }
 
+
+  def directDependencies[T](dag: List[Dag[T]]): List[T] = {
+    dag.foldLeft(List.empty[T]) {
+      case (acc, dag) => dag match {
+        case Leaf(value) => value :: acc
+        case Parent(value, _) => value :: acc
+      }
+    }
+  }
+
   def dfs[T](dag: Dag[T]): List[T] = {
     def loop(dag: Dag[T], acc: List[T]): List[T] = {
       dag match {

frontend/src/main/scala/bloop/engine/Feedback.scala

@@ -13,6 +13,11 @@ object Feedback {
   def expectedMainClass(project: Project): String =
     s"Expected a main class via command-line or in the configuration of project '${project.name}'"
 
+  def missingScalaInstance(project: Project): String =
+    s"Failed to compile project '${project.name}': found Scala sources but project is missing Scala configuration."
+  def missingInstanceForJavaCompilation(project: Project): String =
+    s"Failed to compile Java sources in ${project.name}: default Zinc Scala instance couldn't be created!"
+
   def failedToLink(project: Project, linker: String, t: Throwable): String =
     s"Failed to link $linker project '${project.name}': '${t.getMessage}'"
   def missingLinkArtifactFor(project: Project, artifactName: String, linker: String): String =

frontend/src/main/scala/bloop/engine/Interpreter.scala

@@ -7,7 +7,7 @@ import bloop.cli.completion.{Case, Mode}
 import bloop.config.Config.Platform
 import bloop.io.{AbsolutePath, RelativePath, SourceWatcher}
 import bloop.testing.{LoggingEventHandler, TestInternals}
-import bloop.engine.tasks.{Pipelined, ScalaJsToolchain, ScalaNativeToolchain, Tasks}
+import bloop.engine.tasks.{CompilationTask, ScalaJsToolchain, ScalaNativeToolchain, Tasks}
 import bloop.Project
 import bloop.cli.Commands.{CompilingCommand, LinkingCommand}
 import bloop.config.Config
@@ -133,9 +133,18 @@ object Interpreter {
 
     val compileTask = state.flatMap { state =>
       val config = ReporterKind.toReporterConfig(cmd.reporter)
-      if (cmd.pipelined)
+      /*      if (cmd.pipelined)
         Pipelined.compile(state, project, config, deduplicateFailures, compilerMode, excludeRoot)
-      else Tasks.compile(state, project, config, deduplicateFailures, compilerMode, excludeRoot)
+      else Tasks.compile(state, project, config, deduplicateFailures, compilerMode, excludeRoot)*/
+      CompilationTask.compile(
+        state,
+        project,
+        config,
+        deduplicateFailures,
+        compilerMode,
+        cmd.pipelined,
+        excludeRoot
+      )
     }
 
     compileTask.map(_.mergeStatus(ExitStatus.Ok))
@@ -200,7 +209,7 @@ object Interpreter {
           val cwd = cmd.cliOptions.common.workingPath
           compileAnd(cmd, state, project, false, sequential, "`test`") { state =>
             val handler = new LoggingEventHandler(state.logger)
-            Tasks.test( state, project, cwd, cmd.includeDependencies, cmd.args, testFilter, handler)
+            Tasks.test(state, project, cwd, cmd.includeDependencies, cmd.args, testFilter, handler)
           }
         }
 

frontend/src/main/scala/bloop/engine/caches/ResultsCache.scala

@@ -4,6 +4,7 @@ import java.util.Optional
 
 import bloop.{Compiler, Project}
 import bloop.Compiler.Result
+import bloop.engine.tasks.compilation.FinalCompileResult
 import bloop.engine.{Build, ExecutionContext}
 import bloop.io.AbsolutePath
 import bloop.logging.Logger
@@ -67,6 +68,9 @@ final class ResultsCache private (
   def addResults(ps: List[(Project, Compiler.Result)]): ResultsCache =
     ps.foldLeft(this) { case (rs, (p, r)) => rs.addResult(p, r) }
 
+  def addFinalResults(ps: List[FinalCompileResult]): ResultsCache =
+    ps.foldLeft(this) { case (rs, FinalCompileResult(p, r)) => rs.addResult(p, r) }
+
   override def toString: String = s"ResultsCache(${successful.mkString(", ")})"
 }