Skip to content

Commit

Permalink
Straight to jar compilation
Browse files Browse the repository at this point in the history
  • Loading branch information
lukaszwawrzyk committed Oct 1, 2018
1 parent 64aa612 commit 7b2e998
Show file tree
Hide file tree
Showing 30 changed files with 2,182 additions and 159 deletions.
19 changes: 16 additions & 3 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,12 @@ lazy val zinc = (project in file("zinc"))
.settings(
name := "zinc",
mimaSettings,
mimaBinaryIssueFilters ++= Seq(
exclude[DirectMissingMethodProblem]("sbt.internal.inc.MixedAnalyzingCompiler.searchClasspathAndLookup"),
exclude[IncompatibleResultTypeProblem]("sbt.internal.inc.MixedAnalyzingCompiler.javac"),
exclude[IncompatibleMethTypeProblem]("sbt.internal.inc.MixedAnalyzingCompiler.this"),

)
)

lazy val zincTesting = (project in internalPath / "zinc-testing")
Expand Down Expand Up @@ -281,7 +287,10 @@ lazy val zincCore = (project in internalPath / "zinc-core")
exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.findClassDependencies"),
exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateClassesInternally"),
exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateClassesExternally"),
exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.findAPIChange")
exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.findAPIChange"),
exclude[IncompatibleMethTypeProblem]("sbt.internal.inc.Stamps.initial"),
exclude[IncompatibleMethTypeProblem]("sbt.internal.inc.InitialStamps.this"),
exclude[IncompatibleMethTypeProblem]("sbt.internal.inc.Incremental.prune")
)
}
)
Expand Down Expand Up @@ -391,7 +400,8 @@ lazy val compilerInterface212 = (project in internalPath / "compiler-interface")
exclude[ReversedMissingMethodProblem]("xsbti.compile.ExternalHooks#Lookup.hashClasspath"),
exclude[ReversedMissingMethodProblem]("xsbti.compile.ScalaInstance.loaderLibraryOnly"),
exclude[DirectMissingMethodProblem]("xsbti.api.AnalyzedClass.of"),
exclude[DirectMissingMethodProblem]("xsbti.api.AnalyzedClass.create")
exclude[DirectMissingMethodProblem]("xsbti.api.AnalyzedClass.create"),
exclude[ReversedMissingMethodProblem]("xsbti.compile.analysis.ReadStamps.reset")
)
},
)
Expand Down Expand Up @@ -745,7 +755,10 @@ lazy val zincClassfile212 = zincClassfileTemplate
.settings(
scalaVersion := scala212,
crossScalaVersions := Seq(scala212),
target := (target in zincClassfileTemplate).value.getParentFile / "target-2.12"
target := (target in zincClassfileTemplate).value.getParentFile / "target-2.12",
mimaBinaryIssueFilters ++= Seq(
exclude[DirectMissingMethodProblem]("sbt.internal.inc.classfile.Analyze.apply")
)
)

// re-implementation of scripted engine
Expand Down
9 changes: 7 additions & 2 deletions internal/compiler-bridge/src/main/scala/xsbt/API.scala
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ final class API(val global: CallbackGlobal) extends Compat with GlobalHelpers wi

import scala.collection.mutable
private val nonLocalClassSymbolsInCurrentUnits = new mutable.HashSet[Symbol]()
private val STJ = new STJ(outputDirs)

def newPhase(prev: Phase) = new ApiPhase(prev)
class ApiPhase(prev: Phase) extends GlobalPhase(prev) {
Expand Down Expand Up @@ -96,7 +97,7 @@ final class API(val global: CallbackGlobal) extends Compat with GlobalHelpers wi
*
* This method only takes care of non-local classes because local classes have no
* relevance in the correctness of the algorithm and can be registered after genbcode.
* Local classes are only used to contruct the relations of products and to produce
* Local classes are only used to construct the relations of products and to produce
* the list of generated files + stamps, but names referring to local classes **never**
* show up in the name hashes of classes' APIs, hence never considered for name hashing.
*
Expand All @@ -118,7 +119,11 @@ final class API(val global: CallbackGlobal) extends Compat with GlobalHelpers wi
if (!symbol.isLocalClass) {
val classFileName = s"${names.binaryName}.class"
val outputDir = global.settings.outputDirs.outputDirFor(sourceFile).file
val classFile = new java.io.File(outputDir, classFileName)
val classFile = if (STJ.enabled) {
new java.io.File(STJ.init(outputDir, classFileName))
} else {
new java.io.File(outputDir, classFileName)
}
val zincClassName = names.className
val srcClassName = classNameAsString(symbol)
callback.generatedNonLocalClass(sourceJavaFile, classFile, zincClassName, srcClassName)
Expand Down
54 changes: 45 additions & 9 deletions internal/compiler-bridge/src/main/scala/xsbt/Analyzer.scala
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

package xsbt

import java.io.File

import scala.tools.nsc.Phase

object Analyzer {
Expand All @@ -15,29 +17,44 @@ object Analyzer {

final class Analyzer(val global: CallbackGlobal) extends LocateClassFile {
import global._
private val STJ = new STJ(outputDirs)

def newPhase(prev: Phase): Phase = new AnalyzerPhase(prev)
private class AnalyzerPhase(prev: Phase) extends GlobalPhase(prev) {
override def description =
"Finds concrete instances of provided superclasses, and application entry points."
def name = Analyzer.name

private lazy val existingJaredClasses: Set[STJ.JaredClass] = {
STJ.outputJar
.map { jar =>
val classes = STJ.listFiles(jar)
classes.map(STJ.init(jar, _))
}
.getOrElse(Set.empty)
}

def apply(unit: CompilationUnit): Unit = {
if (!unit.isJava) {
val sourceFile = unit.source.file.file
for (iclass <- unit.icode) {
val sym = iclass.symbol
val outputDir = settings.outputDirs.outputDirFor(sym.sourceFile).file
def addGenerated(separatorRequired: Boolean): Unit = {
val classFile = fileForClass(outputDir, sym, separatorRequired)
if (classFile.exists()) {
assert(sym.isClass, s"${sym.fullName} is not a class")
// Use own map of local classes computed before lambdalift to ascertain class locality
if (localToNonLocalClass.isLocal(sym).getOrElse(true)) {
// Inform callback about local classes, non-local classes have been reported in API
callback.generatedLocalClass(sourceFile, classFile)
}
val locatedClass = if (STJ.enabled) {
locateClassInJar(sym, separatorRequired)
} else {
locatePlainClassFile(sym, separatorRequired)
}

locatedClass
.foreach { classFile =>
assert(sym.isClass, s"${sym.fullName} is not a class")
// Use own map of local classes computed before lambdalift to ascertain class locality
if (localToNonLocalClass.isLocal(sym).getOrElse(true)) {
// Inform callback about local classes, non-local classes have been reported in API
callback.generatedLocalClass(sourceFile, classFile)
}
}
}

if (sym.isModuleClass && !sym.isImplClass) {
Expand All @@ -49,5 +66,24 @@ final class Analyzer(val global: CallbackGlobal) extends LocateClassFile {
}
}
}

private def locatePlainClassFile(sym: Symbol, separatorRequired: Boolean): Option[File] = {
val outputDir = settings.outputDirs.outputDirFor(sym.sourceFile).file
val classFile = fileForClass(outputDir, sym, separatorRequired)
if (classFile.exists()) Some(classFile) else None
}

private def locateClassInJar(sym: Symbol, separatorRequired: Boolean): Option[File] = {
val classFile =
fileForClass(new java.io.File("."), sym, separatorRequired).toString
.drop(2) // stripPrefix ./ or .\
val jaredClass = STJ.init(classFile)
if (existingJaredClasses.contains(jaredClass)) {
Some(new File(jaredClass))
} else {
None
}
}
}

}
23 changes: 19 additions & 4 deletions internal/compiler-bridge/src/main/scala/xsbt/CallbackGlobal.scala
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import scala.tools.nsc._
import io.AbstractFile
import java.io.File

import scala.reflect.io.PlainFile

/** Defines the interface of the incremental compiler hiding implementation details. */
sealed abstract class CallbackGlobal(settings: Settings,
reporter: reporters.Reporter,
Expand Down Expand Up @@ -132,21 +134,34 @@ sealed class ZincCompiler(settings: Settings, dreporter: DelegatingReporter, out
this.computePhaseDescriptors
}

private final val STJ = new STJ(outputDirs)

private final val jaredClassesFromPrevCompilation =
perRunCaches.recordCache(new STJ.PrevJarCache(settings.classpath.value))

private final val fqnsToAssociatedFiles = perRunCaches.newMap[String, (AbstractFile, Boolean)]()

/** Returns the associated file of a fully qualified name and whether it's on the classpath. */
def findAssociatedFile(fqn: String): Option[(AbstractFile, Boolean)] = {
def getOutputClass(name: String): Option[AbstractFile] = {
// This could be improved if a hint where to look is given.
val className = name.replace('.', '/') + ".class"
outputDirs.map(new File(_, className)).find((_.exists)).map((AbstractFile.getFile(_)))
val relPathToClass = name.replace('.', '/') + ".class"
if (STJ.enabled) {
val jaredClass = STJ.init(relPathToClass)
if (jaredClassesFromPrevCompilation.contains(jaredClass)) {
Some(new PlainFile(jaredClass))
} else None
} else {
// This could be improved if a hint where to look is given.
outputDirs.map(new File(_, relPathToClass)).find(_.exists()).map(AbstractFile.getFile(_))
}
}

def findOnClassPath(name: String): Option[AbstractFile] =
classPath.findClass(name).flatMap(_.binary.asInstanceOf[Option[AbstractFile]])

fqnsToAssociatedFiles.get(fqn).orElse {
val newResult = getOutputClass(fqn).map(f => (f, true))
val newResult = getOutputClass(fqn)
.map(f => (f, true))
.orElse(findOnClassPath(fqn).map(f => (f, false)))
newResult.foreach(res => fqnsToAssociatedFiles.put(fqn, res))
newResult
Expand Down
79 changes: 79 additions & 0 deletions internal/compiler-bridge/src/main/scala/xsbt/STJ.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package xsbt
import java.io.File
import java.nio.file.Paths
import java.util.zip.ZipFile

class STJ(outputDirs: Iterable[File]) {
type JaredClass = String
type RelClass = String

def init(jar: File, cls: RelClass): JaredClass = {
// This identifier will be stored as a java.io.File. Its constructor will normalize slashes
// which means that the identifier to be consistent should at all points have consistent
// slashes for safe comparisons, especially in sets or maps.
val relClass = if (File.separatorChar == '/') cls else cls.replace(File.separatorChar, '/')
s"$jar!$relClass"
}

def init(cls: RelClass): JaredClass = {
init(outputJar.get, cls)
}

def listFiles(jar: File): Set[RelClass] = {
import scala.collection.JavaConverters._
// ZipFile is slightly slower than IndexBasedZipFsOps but it is quite difficult to use reuse
// IndexBasedZipFsOps in compiler bridge.
val zip = new ZipFile(jar)
try {
zip.entries().asScala.filterNot(_.isDirectory).map(_.getName).toSet
} finally {
zip.close()
}
}

val outputJar: Option[File] = {
outputDirs match {
case Seq(file) if file.getName.endsWith(".jar") => Some(file)
case _ => None
}
}

val enabled: Boolean = outputJar.isDefined

class PrevJarCache(rawClasspath: String) extends scala.collection.generic.Clearable {
private var cache: Set[JaredClass] = _

private lazy val prevJar = {
val classpath = rawClasspath.split(File.pathSeparator)
findPrevJar(classpath)
}

def contains(jaredClass: JaredClass): Boolean = {
if (cache == null) {
cache = loadEntriesFromPrevJar()
}
cache.contains(jaredClass)
}

def clear(): Unit = cache = null

private def loadEntriesFromPrevJar(): Set[JaredClass] = {
prevJar
.filter(_.exists())
.fold(Set.empty[JaredClass]) { prevJar =>
val classes = listFiles(prevJar)
classes.map(init)
}
}
}

private def findPrevJar(classpath: Seq[String]): Option[File] = {
classpath.headOption.map(new File(_)).filter { path =>
val fileName = path.getName
fileName.startsWith(prevJarPrefix) && fileName.endsWith(".jar")
}
}

private val prevJarPrefix: String = "prev-jar"

}
Original file line number Diff line number Diff line change
Expand Up @@ -72,4 +72,10 @@ public interface ReadStamps {
* @see xsbti.compile.analysis.ReadStamps#product(File)
*/
public Map<File, Stamp> getAllProductStamps();

/**
* Resets internal state of stamp reader that concerns data changing during the compilation.
* Namely it resets the cache for compilation products. Should be called before each compilation.
*/
public void reset();
}
Loading

0 comments on commit 7b2e998

Please sign in to comment.