Add support for linker and optimizer reporters (#405)

Reporters supersede harcoded `nativeEmitDependencyGraph`
and `nativeVerbose` options that were used to debug the
scala native toolchain before.

build.sbt

@@ -1,4 +1,6 @@
 import scala.util.Try
+import scalanative.tools.OptimizerReporter
+import scalanative.sbtplugin.ScalaNativePluginInternal.nativeOptimizerReporter
 
 val toolScalaVersion = "2.10.6"
 
@@ -112,7 +114,6 @@ lazy val libSettings =
 lazy val projectSettings =
   ScalaNativePlugin.projectSettings ++ Seq(
     scalaVersion := libScalaVersion,
-    nativeVerbose := true,
     nativeClangOptions ++= Seq("-O0"),
     resolvers := Nil
   )
@@ -329,6 +330,10 @@ lazy val sandbox =
     .in(file("sandbox"))
     .settings(projectSettings)
     .settings(noPublishSettings)
+    .settings(
+      nativeOptimizerReporter := OptimizerReporter.toDirectory(
+        crossTarget.value)
+    )
     .enablePlugins(ScalaNativePlugin)
 
 lazy val benchmarks =

sbt-scala-native/src/main/scala/scala/scalanative/sbtplugin/ScalaNativePlugin.scala

@@ -1,6 +1,8 @@
 package scala.scalanative
 package sbtplugin
 
+import scalanative.tools
+
 import sbt._
 
 object ScalaNativePlugin extends AutoPlugin {
@@ -14,18 +16,13 @@ object ScalaNativePlugin extends AutoPlugin {
 
     val nativeVersion = nir.Versions.current
 
-    val nativeVerbose = settingKey[Boolean]("Enable verbose tool logging.")
-
     val nativeClang = settingKey[File]("Location of the clang compiler.")
 
     val nativeClangPP = settingKey[File]("Location of the clang++ compiler.")
 
     val nativeClangOptions =
       settingKey[Seq[String]]("Additional options that are passed to clang.")
 
-    val nativeEmitDependencyGraphPath = settingKey[Option[File]](
-      "If non-empty, emit linker graph to the given file path.")
-
     val nativeLibraryLinkage = settingKey[Map[String, String]](
       "Given a native library, provide the linkage kind (static or dynamic). " +
         "If key is not present in the map, dynamic is picked as a default.")

sbt-scala-native/src/main/scala/scala/scalanative/sbtplugin/ScalaNativePluginInternal.scala

@@ -19,6 +19,12 @@ import scala.util.Try
 import System.{lineSeparator => nl}
 
 object ScalaNativePluginInternal {
+  val nativeLinkerReporter = settingKey[tools.LinkerReporter](
+    "A reporter that gets notified whenever a linking event happens.")
+
+  val nativeOptimizerReporter = settingKey[tools.OptimizerReporter](
+    "A reporter that gets notified whenever an optimizer event happens.")
+
   private lazy val nativelib: File =
     Path.userHome / ".scalanative" / ("nativelib-" + nir.Versions.current)
 
@@ -68,23 +74,29 @@ object ScalaNativePluginInternal {
     }
   }
 
-  /** Compiles application nir to llvm ir. */
-  private def compileNir(config: tools.Config,
-                         logger: Logger): Seq[nir.Attr.Link] = {
-    val driver                   = tools.Driver(config)
-    val (unresolved, links, raw) = tools.link(config, driver)
+  private def reportLinkingErrors(unresolved: Seq[nir.Global],
+                                  logger: Logger): Nothing = {
+    import nir.Shows._
 
-    if (!unresolved.isEmpty) {
-      import nir.Shows._
+    unresolved.map(u => sh"$u".toString).sorted.foreach { signature =>
+      logger.error(s"cannot link: $signature")
+    }
 
-      unresolved.map(u => sh"$u".toString).sorted.foreach { signature =>
-        logger.error(s"cannot link: $signature")
-      }
+    throw new MessageOnlyException("unable to link")
+  }
 
-      throw new MessageOnlyException("unable to link")
-    }
+  /** Compiles application nir to llvm ir. */
+  private def compileNir(
+      config: tools.Config,
+      logger: Logger,
+      linkerReporter: tools.LinkerReporter,
+      optimizerReporter: tools.OptimizerReporter): Seq[nir.Attr.Link] = {
+    val driver                   = tools.OptimizerDriver(config)
+    val (unresolved, links, raw) = tools.link(config, driver, linkerReporter)
+
+    if (unresolved.nonEmpty) { reportLinkingErrors(unresolved, logger) }
 
-    val optimized = tools.optimize(config, driver, raw)
+    val optimized = tools.optimize(config, driver, raw, optimizerReporter)
     tools.codegen(config, optimized)
 
     links
@@ -178,8 +190,6 @@ object ScalaNativePluginInternal {
     ),
     addCompilerPlugin(
       "org.scala-native" % "nscplugin" % nativeVersion cross CrossVersion.full),
-    nativeVerbose := false,
-    nativeEmitDependencyGraphPath := None,
     nativeLibraryLinkage := Map(),
     nativeSharedLibrary := false,
     nativeClang := {
@@ -200,30 +210,32 @@ object ScalaNativePluginInternal {
     artifactPath in nativeLink := {
       (crossTarget in Compile).value / (moduleName.value + "-out")
     },
+    nativeLinkerReporter := tools.LinkerReporter.empty,
+    nativeOptimizerReporter := tools.OptimizerReporter.empty,
     nativeLink := ResourceScope { implicit in =>
       val mainClass = (selectMainClass in Compile).value.getOrElse(
         throw new MessageOnlyException("No main class detected.")
       )
-      val entry         = nir.Global.Top(mainClass.toString + "$")
-      val classpath     = (fullClasspath in Compile).value.map(_.data)
-      val target        = (crossTarget in Compile).value
-      val appll         = target / "out.ll"
-      val binary        = (artifactPath in nativeLink).value
-      val clang         = nativeClang.value
-      val clangpp       = nativeClangPP.value
-      val clangOpts     = nativeClangOptions.value
-      val dotpath       = nativeEmitDependencyGraphPath.value
-      val linkage       = nativeLibraryLinkage.value
-      val sharedLibrary = nativeSharedLibrary.value
-      val logger        = streams.value.log
+      val entry             = nir.Global.Top(mainClass.toString + "$")
+      val classpath         = (fullClasspath in Compile).value.map(_.data)
+      val target            = (crossTarget in Compile).value
+      val appll             = target / "out.ll"
+      val binary            = (artifactPath in nativeLink).value
+      val clang             = nativeClang.value
+      val clangpp           = nativeClangPP.value
+      val clangOpts         = nativeClangOptions.value
+      val linkage           = nativeLibraryLinkage.value
+      val linkerReporter    = nativeLinkerReporter.value
+      val optimizerReporter = nativeOptimizerReporter.value
+      val sharedLibrary     = nativeSharedLibrary.value
+      val logger            = streams.value.log
 
       val config = tools.Config.empty
         .withEntry(entry)
-        .withPaths(classpath.map(p => tools.Path(VirtualDirectory.real(p))))
+        .withPaths(classpath.map(p =>
+          tools.LinkerPath(VirtualDirectory.real(p))))
         .withTargetDirectory(VirtualDirectory.real(target))
         .withInjectMain(!nativeSharedLibrary.value)
-        .withCheck(false)
-        .withVerbose(nativeVerbose.value)
 
       checkThatClangIsRecentEnough(clang)
 
@@ -252,12 +264,13 @@ object ScalaNativePluginInternal {
               unpackNativelib(clang, clangpp, classpath, logger)
 
             if (unpackSuccess) {
-              val links = compileNir(config, logger).map(_.name)
+              val links =
+                compileNir(config, logger, linkerReporter, optimizerReporter)
               compileLl(clangpp,
                         target,
                         appll,
                         binary,
-                        links,
+                        links.map(_.name),
                         linkage,
                         clangOpts,
                         logger)

scripted-tests/run/linker-reporter/build.sbt

@@ -0,0 +1,14 @@
+import scalanative.tools.LinkerReporter
+import scalanative.sbtplugin.ScalaNativePluginInternal.nativeLinkerReporter
+
+ScalaNativePlugin.projectSettings
+
+scalaVersion := "2.11.8"
+
+nativeLinkerReporter := LinkerReporter.toFile(target.value / "out.dot")
+
+lazy val check = taskKey[Unit]("Check that dot file was created.")
+
+check := {
+  assert((target.value / "out.dot").exists)
+}

scripted-tests/run/linker-reporter/project/scala-native.sbt

@@ -0,0 +1,10 @@
+{
+  val pluginVersion = System.getProperty("plugin.version")
+  if (pluginVersion == null)
+    throw new RuntimeException(
+      """|The system property 'plugin.version' is not defined.
+         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin)
+  else addSbtPlugin("org.scala-native" % "sbt-scala-native" % pluginVersion)
+}
+
+resolvers += Resolver.sonatypeRepo("snapshots")

scripted-tests/run/linker-reporter/src/main/scala/Hello.scala

@@ -0,0 +1,3 @@
+object Main {
+  def main(args: Array[String]): Unit = ()
+}

scripted-tests/run/linker-reporter/test

@@ -0,0 +1,2 @@
+> nativeLink
+> check

scripted-tests/run/optimizer-reporter/build.sbt

@@ -0,0 +1,14 @@
+import scalanative.tools.OptimizerReporter
+import scalanative.sbtplugin.ScalaNativePluginInternal.nativeOptimizerReporter
+
+ScalaNativePlugin.projectSettings
+
+scalaVersion := "2.11.8"
+
+nativeOptimizerReporter := OptimizerReporter.toDirectory(crossTarget.value)
+
+lazy val check = taskKey[Unit]("Check that dot file was created.")
+
+check := {
+  assert((crossTarget.value / "out.00.hnir").exists)
+}

scripted-tests/run/optimizer-reporter/project/scala-native.sbt

@@ -0,0 +1,10 @@
+{
+  val pluginVersion = System.getProperty("plugin.version")
+  if (pluginVersion == null)
+    throw new RuntimeException(
+      """|The system property 'plugin.version' is not defined.
+         |Specify this property using the scriptedLaunchOpts -D.""".stripMargin)
+  else addSbtPlugin("org.scala-native" % "sbt-scala-native" % pluginVersion)
+}
+
+resolvers += Resolver.sonatypeRepo("snapshots")

scripted-tests/run/optimizer-reporter/src/main/scala/Hello.scala

@@ -0,0 +1,3 @@
+object Main {
+  def main(args: Array[String]): Unit = ()
+}

scripted-tests/run/optimizer-reporter/test

@@ -0,0 +1,2 @@
+> nativeLink
+> check

tools/src/main/scala/scala/scalanative/linker/Linker.scala

@@ -17,9 +17,14 @@ sealed trait Linker {
 object Linker {
 
   /** Create a new linker given tools configuration. */
-  def apply(config: tools.Config): Linker = new Impl(config)
+  def apply(config: tools.Config,
+            reporter: Reporter = Reporter.empty): Linker =
+    new Impl(config, reporter)
+
+  private final class Impl(config: tools.Config, reporter: Reporter)
+      extends Linker {
+    import reporter._
 
-  private final class Impl(config: tools.Config) extends Linker {
     private def load(
         global: Global): Option[(Seq[Dep], Seq[Attr.Link], Defn)] =
       config.paths.collectFirst {
@@ -44,18 +49,22 @@ object Linker {
 
             load(workitem).fold[Unit] {
               unresolved += workitem
+              onUnresolved(workitem)
             } {
               case (deps, newlinks, defn) =>
                 resolved += workitem
                 defns += defn
                 links ++= newlinks
+                onResolved(workitem)
 
                 deps.foreach {
                   case Dep.Direct(dep) =>
                     direct.push(dep)
+                    onDirectDependency(workitem, dep)
 
-                  case cond: Dep.Conditional =>
+                  case cond @ Dep.Conditional(dep, condition) =>
                     conditional += cond
+                    onConditionalDependency(workitem, dep, condition)
                 }
             }
           }
@@ -79,12 +88,20 @@ object Linker {
         conditional = rest
       }
 
-      direct.pushAll(entries)
+      onStart()
+
+      entries.foreach { entry =>
+        direct.push(entry)
+        onEntry(entry)
+      }
+
       while (direct.nonEmpty) {
         processDirect
         processConditional
       }
 
+      onComplete()
+
       (unresolved.toSeq, links.toSeq, defns.sortBy(_.name.toString).toSeq)
     }
   }

tools/src/main/scala/scala/scalanative/linker/Reporter.scala

@@ -0,0 +1,78 @@
+package scala.scalanative
+package linker
+
+import java.io.{File, PrintWriter}
+import nir.Global
+import nir.Shows._
+import util.sh
+
+/** Linking reporters can override one of the corresponding methods to
+ *  get notified whenever one of the linking events happens.
+ */
+trait Reporter {
+
+  /** Gets called whenever linking starts. */
+  def onStart(): Unit = ()
+
+  /** Gets called whenever a new entry point is discovered. */
+  def onEntry(global: Global): Unit = ()
+
+  /** Gets called whenever a new definition is loaded from nir path. */
+  def onResolved(global: Global): Unit = ()
+
+  /** Gets called whenever linker fails to resolve a global. */
+  def onUnresolved(globa: Global): Unit = ()
+
+  /** Gets called whenever a new direct dependency is discovered. */
+  def onDirectDependency(from: Global, to: Global): Unit = ()
+
+  /** Gets called whenever a new conditional dependency is discovered. */
+  def onConditionalDependency(from: Global, to: Global, cond: Global): Unit =
+    ()
+
+  /** Gets called whenever linking is complete. */
+  def onComplete(): Unit = ()
+}
+
+object Reporter {
+
+  /** Default no-op reporter. */
+  val empty = new Reporter {}
+
+  /** Generate dot file for observed dependency graph. */
+  def toFile(file: File): Reporter = new Reporter {
+    private var writer: PrintWriter = _
+
+    private def writeStart(): Unit = {
+      writer = new PrintWriter(file)
+      writer.println("digraph G {")
+    }
+
+    private def writeEdge(from: Global, to: Global): Unit = {
+      def quoted(s: String) = "\"" + s + "\""
+      writer.print(quoted(sh"$from".toString))
+      writer.print("->")
+      writer.print(quoted(sh"$to".toString))
+      writer.println(";")
+    }
+
+    private def writeEnd(): Unit = {
+      writer.println("}")
+      writer.close()
+    }
+
+    override def onStart(): Unit =
+      writeStart()
+
+    override def onDirectDependency(from: Global, to: Global): Unit =
+      writeEdge(from, to)
+
+    override def onConditionalDependency(from: Global,
+                                         to: Global,
+                                         cond: Global): Unit =
+      writeEdge(from, to)
+
+    override def onComplete(): Unit =
+      writeEnd()
+  }
+}