Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Better backtrace in classloading for unreachable symbols #3449

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import _root_.sbt.testing._
import java.net.URLClassLoader
import java.io.File
import scala.scalanative.build.Build
import scala.scalanative.linker.Result
import scala.scalanative.linker.ReachabilityAnalysis
import scala.concurrent._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
Expand Down Expand Up @@ -150,19 +150,17 @@ case class PartestTask(taskDef: TaskDef, args: Array[String]) extends Task {
}

import scala.collection.mutable
val linkerResult = new Result(
val analysis = new ReachabilityAnalysis.Result(
infos = mutable.Map.empty,
entries = Nil,
unavailable = Nil,
referencedFrom = mutable.Map.empty,
links = Defaults.links,
defns = Nil,
dynsigs = Nil,
dynimpls = Nil,
resolvedVals = mutable.Map.empty
)

val build = Build.findAndCompileNativeLibs(config, linkerResult)
val build = Build.findAndCompileNativeLibs(config, analysis)
Await.result(build, Duration.Inf)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import scala.scalanative.build._
import scala.concurrent._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.scalanative.linker.ReachabilityAnalysis

@Fork(1)
@State(Scope.Benchmark)
Expand All @@ -22,7 +23,7 @@ import scala.concurrent.duration._
@Measurement(iterations = 10, time = 2, timeUnit = TimeUnit.SECONDS)
abstract class CodeGenBench(nativeConfig: NativeConfig => NativeConfig) {
var config: Config = _
var linked: linker.Result = _
var analysis: ReachabilityAnalysis.Result = _

@Setup(Level.Trial)
def setup(): Unit = {
Expand All @@ -35,7 +36,7 @@ abstract class CodeGenBench(nativeConfig: NativeConfig => NativeConfig) {

val entries = build.ScalaNative.entries(config)
util.Scope { implicit scope =>
linked = Await.result(
analysis = Await.result(
ScalaNative.link(config, entries),
Duration.Inf
)
Expand All @@ -49,13 +50,13 @@ abstract class CodeGenBench(nativeConfig: NativeConfig => NativeConfig) {
.walk(workdir)
.sorted(Comparator.reverseOrder())
.forEach(Files.delete)
linked = null
analysis = null
config = null
}

@Benchmark
def codeGen(): Unit = {
val codegen = ScalaNative.codegen(config, linked)
val codegen = ScalaNative.codegen(config, analysis)
val paths = Await.result(codegen, Duration.Inf)
assert(paths.nonEmpty)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ class LinkerBench {

val entries = build.ScalaNative.entries(config)
val link = build.ScalaNative.link(config, entries)
val linked = Await.result(link, Duration.Inf)
assert(linked.unavailable.size == 0)
Await.result(link, Duration.Inf)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import scala.scalanative.build._
import scala.concurrent._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.scalanative.linker.ReachabilityAnalysis

@Fork(1)
@State(Scope.Benchmark)
Expand All @@ -21,7 +22,7 @@ import scala.concurrent.duration._
@Measurement(iterations = 10, time = 2, timeUnit = TimeUnit.SECONDS)
abstract class OptimizerBench(mode: build.Mode) {
var config: Config = _
var linked: linker.Result = _
var analysis: ReachabilityAnalysis.Result = _

@Setup(Level.Trial)
def setup(): Unit = {
Expand All @@ -33,7 +34,7 @@ abstract class OptimizerBench(mode: build.Mode) {

val entries = build.ScalaNative.entries(config)
util.Scope { implicit scope =>
linked = Await.result(
analysis = Await.result(
ScalaNative.link(config, entries),
Duration.Inf
)
Expand All @@ -47,19 +48,19 @@ abstract class OptimizerBench(mode: build.Mode) {
.walk(workdir)
.sorted(Comparator.reverseOrder())
.forEach(Files.delete)
linked = null
analysis = null
config = null
}

@Benchmark
def optimize(): Unit = {
val optimize = ScalaNative.optimize(config, linked)
val optimize = ScalaNative.optimize(config, analysis)
val optimized = Await.result(optimize, Duration.Inf)
assert(optimized.unavailable.size == 0)
}
}

class OptimizeDebug extends OptimizerBench(build.Mode.debug)
class OptimizeReleaseFast extends OptimizerBench(build.Mode.releaseFast)

// Commented out becouse of long build times ~13 min
// class OptimizeReleaseFull extends OptimizerBench(build.Mode.releaseFull)
21 changes: 10 additions & 11 deletions tools/src/main/scala/scala/scalanative/build/Build.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package build

import java.nio.file.{Files, Path, Paths}
import scala.scalanative.util.Scope
import scala.scalanative.linker.ReachabilityAnalysis
import scala.util.Try
import java.nio.file.FileVisitOption
import java.util.Optional
Expand Down Expand Up @@ -107,11 +108,6 @@ object Build {
config.logger.debug(config.toString())

link(config, entries(config))
.map { linked =>
// Can throw, execute in the main flow
logLinked(config, linked)
linked
}
.flatMap(optimize(config, _))
.flatMap { linkerResult =>
val backend = new BackendPipeline(config, linkerResult)
Expand All @@ -124,13 +120,16 @@ object Build {
}
}

private class BackendPipeline(config: Config, linkerResult: linker.Result) {
private class BackendPipeline(
config: Config,
analysis: ReachabilityAnalysis.Result
) {
private val logger = config.logger
import logger._

def codegen()(implicit ec: ExecutionContext): Future[Seq[Path]] = {
val tasks = immutable.Seq(
ScalaNative.codegen(config, linkerResult),
ScalaNative.codegen(config, analysis),
genBuildInfo(config)
)
Future.reduceLeft(tasks)(_ ++ _)
Expand All @@ -152,7 +151,7 @@ object Build {
/* Finds all the libraries on the classpath that contain native
* code and then compiles them.
*/
findAndCompileNativeLibs(config, linkerResult)
findAndCompileNativeLibs(config, analysis)
case Some(libObjectPaths) =>
Future.successful {
libObjectPaths
Expand All @@ -171,7 +170,7 @@ object Build {
def link(compiled: Seq[Path]): Path = time(
s"Linking native code (${config.gc.name} gc, ${config.LTO.name} lto)"
) {
LLVM.link(config, linkerResult, compiled)
LLVM.link(config, analysis, compiled)
}

def postProcess(artifact: Path): Path = time(
Expand Down Expand Up @@ -209,12 +208,12 @@ object Build {
*/
def findAndCompileNativeLibs(
config: Config,
linkerResult: linker.Result
analysis: ReachabilityAnalysis.Result
)(implicit ec: ExecutionContext): Future[Seq[Path]] = {
import NativeLib.{findNativeLibs, compileNativeLibrary}
Future
.traverse(findNativeLibs(config))(
compileNativeLibrary(config, linkerResult, _)
compileNativeLibrary(config, analysis, _)
)
.map(_.flatten)
}
Expand Down
5 changes: 3 additions & 2 deletions tools/src/main/scala/scala/scalanative/build/Filter.scala
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import java.nio.file.{Files, Path, Paths}

import scalanative.build.IO.RichPath
import scalanative.build.NativeLib._
import scala.scalanative.linker.ReachabilityAnalysis

private[scalanative] object Filter {

Expand All @@ -27,7 +28,7 @@ private[scalanative] object Filter {
*/
def filterNativelib(
config: Config,
linkerResult: linker.Result,
analysis: ReachabilityAnalysis.Result,
destPath: Path,
allPaths: Seq[Path]
): (Seq[Path], Config) = {
Expand All @@ -43,7 +44,7 @@ private[scalanative] object Filter {
def include(path: String) = {
if (path.contains(optPath)) {
val name = Paths.get(path).toFile.getName.split("\\.").head
linkerResult.links.exists(_.name == name)
analysis.links.exists(_.name == name)
} else {
true
}
Expand Down
9 changes: 5 additions & 4 deletions tools/src/main/scala/scala/scalanative/build/LLVM.scala
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import java.io.{File, PrintWriter}
import java.nio.file.{Files, Path, Paths, StandardCopyOption}
import scala.sys.process._
import scala.scalanative.build.IO.RichPath
import scala.scalanative.linker.ReachabilityAnalysis
import scala.scalanative.nir.Attr.Link

import scala.concurrent._
Expand Down Expand Up @@ -126,7 +127,7 @@ private[scalanative] object LLVM {
*/
def link(
config: Config,
linkerResult: linker.Result,
analysis: ReachabilityAnalysis.Result,
objectsPaths: Seq[Path]
): Path = {
implicit val _config: Config = config
Expand All @@ -139,7 +140,7 @@ private[scalanative] object LLVM {

val command = config.compilerConfig.buildTarget match {
case BuildTarget.Application | BuildTarget.LibraryDynamic =>
prepareLinkCommand(objectsPaths, linkerResult)
prepareLinkCommand(objectsPaths, analysis)
case BuildTarget.LibraryStatic =>
prepareArchiveCommand(objectsPaths)
}
Expand Down Expand Up @@ -179,11 +180,11 @@ private[scalanative] object LLVM {

private def prepareLinkCommand(
objectsPaths: Seq[Path],
linkerResult: linker.Result
analysis: ReachabilityAnalysis.Result
)(implicit config: Config) = {
val workDir = config.workDir
val links = {
val srclinks = linkerResult.links.collect {
val srclinks = analysis.links.collect {
case Link("z") if config.targetsWindows => "zlib"
case Link(name) => name
}
Expand Down
5 changes: 3 additions & 2 deletions tools/src/main/scala/scala/scalanative/build/NativeLib.scala
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import java.nio.file.{Files, Path}
import java.util.Arrays
import java.util.regex._
import scala.concurrent._
import scala.scalanative.linker.ReachabilityAnalysis

/** Original jar or dir path and generated dir path for native code */
private[scalanative] case class NativeLib(src: Path, dest: Path)
Expand All @@ -30,13 +31,13 @@ private[scalanative] object NativeLib {
*/
def compileNativeLibrary(
config: Config,
linkerResult: linker.Result,
analysis: ReachabilityAnalysis.Result,
nativeLib: NativeLib
)(implicit ec: ExecutionContext): Future[Seq[Path]] = {
val destPath = NativeLib.unpackNativeCode(nativeLib)
val paths = NativeLib.findNativePaths(config.workDir, destPath)
val (projPaths, projConfig) =
Filter.filterNativelib(config, linkerResult, destPath, paths)
Filter.filterNativelib(config, analysis, destPath, paths)
LLVM.compile(projConfig, projPaths)
}

Expand Down