Skip to content

Commit

Permalink
Integrate compiler bridge sources with git history from zinc
Browse files Browse the repository at this point in the history
```
➜ scala13 git log --oneline -n1 | cat
b913236 Merge pull request scala#10399 from lrytz/t12774

➜ scala13 git checkout -b sbt-bridge-2023
Switched to a new branch 'sbt-bridge-2023'

➜ scala13 git remote add zinc-bridge-only ../zinc-bridge-only
➜ scala13 git fetch zinc-bridge-only

➜ scala13 git merge --allow-unrelated-histories zinc-bridge-only/bridge-only
```
  • Loading branch information
lrytz committed Jul 12, 2023
2 parents b913236 + 80e8df9 commit b8cebcc
Show file tree
Hide file tree
Showing 29 changed files with 3,873 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
xsbt.InteractiveConsoleBridgeFactory
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
xsbt.CompilerBridge
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
xsbt.ConsoleBridge
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
xsbt.ScaladocBridge
218 changes: 218 additions & 0 deletions src/sbt-bridge/scala/tools/xsbt/API.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
/*
* Zinc - The incremental compiler for Scala.
* Copyright Scala Center, Lightbend, and Mark Harrah
*
* Licensed under Apache License 2.0
* SPDX-License-Identifier: Apache-2.0
*
* See the NOTICE file distributed with this work for
* additional information regarding copyright ownership.
*/

package xsbt

import scala.tools.nsc.Phase
import scala.tools.nsc.symtab.Flags
import xsbti.api._
import xsbti.VirtualFile

object API {
val name = "xsbt-api"
}

final class API(val global: CallbackGlobal) extends Compat with GlobalHelpers with ClassName {
import global._

import scala.collection.mutable
private val nonLocalClassSymbolsInCurrentUnits = new mutable.HashSet[Symbol]()

def newPhase(prev: Phase) = new ApiPhase(prev)
class ApiPhase(prev: Phase) extends GlobalPhase(prev) {
override def description = "Extracts the public API from source files."
def name = API.name
override def run(): Unit = {
val start = System.currentTimeMillis
super.run()

// After processing all units, register generated classes
registerGeneratedClasses(nonLocalClassSymbolsInCurrentUnits.iterator)
nonLocalClassSymbolsInCurrentUnits.clear()

callback.apiPhaseCompleted()
val stop = System.currentTimeMillis
debuglog("API phase took : " + ((stop - start) / 1000.0) + " s")
}

// TODO In 2.13, shouldSkipThisPhaseForJava should be overridden instead of cancelled
// override def shouldSkipThisPhaseForJava = !global.callback.isPickleJava
override def cancelled(unit: CompilationUnit) = {
if (Thread.interrupted()) reporter.cancelled = true
reporter.cancelled || unit.isJava && !global.callback.isPickleJava
}

def apply(unit: global.CompilationUnit): Unit = processUnit(unit)

private def processUnit(unit: CompilationUnit): Unit = {
if (!unit.isJava || global.callback.isPickleJava) {
processScalaUnit(unit)
}
}

private def processScalaUnit(unit: CompilationUnit): Unit = {
val sourceFile: VirtualFile = unit.source.file match { case AbstractZincFile(vf) => vf }
debuglog("Traversing " + sourceFile)
callback.startSource(sourceFile)
val extractApi = new ExtractAPI[global.type](global, sourceFile)
val traverser = new TopLevelHandler(extractApi)
traverser.apply(unit.body)

val extractUsedNames = new ExtractUsedNames[global.type](global)
extractUsedNames.extractAndReport(unit)

val classApis = traverser.allNonLocalClasses
val mainClasses = traverser.mainClasses

// Use of iterators make this code easier to profile

val classApisIt = classApis.iterator
while (classApisIt.hasNext) {
callback.api(sourceFile, classApisIt.next())
}

val mainClassesIt = mainClasses.iterator
while (mainClassesIt.hasNext) {
callback.mainClass(sourceFile, mainClassesIt.next())
}

extractApi.allExtractedNonLocalSymbols.foreach { cs =>
// Only add the class symbols defined in this compilation unit
if (cs.sourceFile != null) nonLocalClassSymbolsInCurrentUnits.+=(cs)
}
}
}

private case class FlattenedNames(binaryName: String, className: String)

/**
* Registers only non-local generated classes in the callback by extracting
* information about its names and using the names to generate class file paths.
*
* Mimics the previous logic that was present in `Analyzer`, despite the fact
* that now we construct the names that the compiler will give to every non-local
* class independently of genbcode.
*
* Why do we do this? The motivation is that we want to run the incremental algorithm
* independently of the compiler pipeline. This independence enables us to:
*
* 1. Offload the incremental compiler logic out of the primary pipeline and
* run the incremental phases concurrently.
* 2. Know before the compilation is completed whether another compilation will or
* will not be required. This is important to make incremental compilation work
* with pipelining and enables further optimizations; for example, we can start
* subsequent incremental compilations before (!) the initial compilation is done.
* This can buy us ~30-40% faster incremental compiler iterations.
*
* This method only takes care of non-local classes because local classes have no
* relevance in the correctness of the algorithm and can be registered after genbcode.
* Local classes are only used to construct the relations of products and to produce
* the list of generated files + stamps, but names referring to local classes **never**
* show up in the name hashes of classes' APIs, hence never considered for name hashing.
*
* As local class files are owned by other classes that change whenever they change,
* we could most likely live without adding their class files to the products relation
* and registering their stamps. However, to be on the safe side, we will continue to
* register the local products in `Analyzer`.
*
* @param allClassSymbols The class symbols found in all the compilation units.
*/
def registerGeneratedClasses(classSymbols: Iterator[Symbol]): Unit = {
classSymbols.foreach { symbol =>
val sourceFile = symbol.sourceFile
val sourceVF0 =
if (sourceFile == null) symbol.enclosingTopLevelClass.sourceFile
else sourceFile
val sourceVF: Option[VirtualFile] = sourceVF0 match {
case AbstractZincFile(vf) => Some(vf)
// This could be scala.reflect.io.FileZipArchive$LeakyEntry
case _ => None
}

def registerProductNames(names: FlattenedNames): Unit = {
// Guard against a local class in case it surreptitiously leaks here
if (!symbol.isLocalClass) {
val pathToClassFile = s"${names.binaryName}.class"
val classFile = {
JarUtils.outputJar match {
case Some(outputJar) =>
new java.io.File(JarUtils.classNameInJar(outputJar, pathToClassFile))
case None =>
val outputDir = global.settings.outputDirs.outputDirFor(sourceFile).file
new java.io.File(outputDir, pathToClassFile)
}
}
val zincClassName = names.className
val srcClassName = classNameAsString(symbol)
sourceVF foreach { source =>
callback.generatedNonLocalClass(
source,
classFile.toPath,
zincClassName,
srcClassName
)
}
} else ()
}

val names = FlattenedNames(
fullName(symbol, java.io.File.separatorChar, symbol.moduleSuffix, true),
fullName(symbol, '.', symbol.moduleSuffix, false)
)

registerProductNames(names)

// Register the names of top-level module symbols that emit two class files
val isTopLevelUniqueModule =
symbol.owner.isPackageClass && symbol.isModuleClass && symbol.companionClass == NoSymbol
if (isTopLevelUniqueModule || symbol.isPackageObject) {
val names = FlattenedNames(
fullName(symbol, java.io.File.separatorChar, "", true),
fullName(symbol, '.', "", false)
)
registerProductNames(names)
}
}
}

private final class TopLevelHandler(extractApi: ExtractAPI[global.type])
extends TopLevelTraverser {
def allNonLocalClasses: Set[ClassLike] = {
extractApi.allExtractedNonLocalClasses
}

def mainClasses: Set[String] = extractApi.mainClasses

def `class`(c: Symbol): Unit = {
extractApi.extractAllClassesOf(c.owner, c)
}
}

private abstract class TopLevelTraverser extends Traverser {
def `class`(s: Symbol): Unit
override def traverse(tree: Tree): Unit = {
tree match {
case (_: ClassDef | _: ModuleDef) if isTopLevel(tree.symbol) => `class`(tree.symbol)
case _: PackageDef =>
super.traverse(tree)
case _ =>
}
}
def isTopLevel(sym: Symbol): Boolean = {
!ignoredSymbol(sym) &&
sym.isStatic &&
!sym.isImplClass &&
(!sym.hasFlag(Flags.JAVA) || global.callback.isPickleJava) &&
!sym.isNestedClass
}
}

}
41 changes: 41 additions & 0 deletions src/sbt-bridge/scala/tools/xsbt/AbstractZincFile.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Zinc - The incremental compiler for Scala.
* Copyright Scala Center, Lightbend, and Mark Harrah
*
* Licensed under Apache License 2.0
* SPDX-License-Identifier: Apache-2.0
*
* See the NOTICE file distributed with this work for
* additional information regarding copyright ownership.
*/

package xsbt

import xsbti.{ PathBasedFile, VirtualFile }
import scala.reflect.io.Streamable

private trait AbstractZincFile extends scala.reflect.io.AbstractFile {
def underlying: VirtualFile
}

private final class ZincPlainFile private[xsbt] (val underlying: PathBasedFile)
extends scala.reflect.io.PlainFile(scala.reflect.io.Path(underlying.toPath.toFile))
with AbstractZincFile

private final class ZincVirtualFile private[xsbt] (val underlying: VirtualFile)
extends scala.reflect.io.VirtualFile(underlying.name, underlying.id)
with AbstractZincFile {
Streamable.closing(output)(_.write(Streamable.bytes(underlying.input))) // fill in the content
}

private object AbstractZincFile {
def apply(virtualFile: VirtualFile): AbstractZincFile = virtualFile match {
case file: PathBasedFile => new ZincPlainFile(file)
case _ => new ZincVirtualFile(virtualFile)
}

def unapply(file: scala.reflect.io.AbstractFile): Option[VirtualFile] = file match {
case wrapper: AbstractZincFile => Some(wrapper.underlying)
case _ => None
}
}
102 changes: 102 additions & 0 deletions src/sbt-bridge/scala/tools/xsbt/Analyzer.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
/*
* Zinc - The incremental compiler for Scala.
* Copyright Scala Center, Lightbend, and Mark Harrah
*
* Licensed under Apache License 2.0
* SPDX-License-Identifier: Apache-2.0
*
* See the NOTICE file distributed with this work for
* additional information regarding copyright ownership.
*/

package xsbt

import java.nio.file.Path
import java.io.File
import xsbti.VirtualFile
import scala.tools.nsc.Phase
import scala.collection.JavaConverters._

object Analyzer {
def name = "xsbt-analyzer"
}

final class Analyzer(val global: CallbackGlobal) extends LocateClassFile {
import global._

def newPhase(prev: Phase): Phase = new AnalyzerPhase(prev)
private class AnalyzerPhase(prev: Phase) extends GlobalPhase(prev) {
override def description =
"Finds concrete instances of provided superclasses, and application entry points."
def name = Analyzer.name

/**
* When straight-to-jar compilation is enabled, returns the classes
* that are found in the jar of the last compilation. This method
* gets the existing classes from the analysis callback and adapts
* it for consumption in the compiler bridge.
*
* It's lazy because it triggers a read of the zip, which may be
* unnecessary if there are no local classes in a compilation unit.
*/
private lazy val classesWrittenByGenbcode: Set[String] = {
JarUtils.outputJar match {
case Some(jar) =>
val classes = global.callback.classesInOutputJar().asScala
classes.map(JarUtils.classNameInJar(jar, _)).toSet
case None => Set.empty
}
}

def apply(unit: CompilationUnit): Unit = {
if (!unit.isJava) {
val sourceFile0: AbstractZincFile = unit.source.file match { case v: AbstractZincFile => v }
val sourceFile: VirtualFile = sourceFile0.underlying
lazy val outputDir = settings.outputDirs.outputDirFor(sourceFile0).file
for (iclass <- unit.icode) {
val sym = iclass.symbol
def addGenerated(separatorRequired: Boolean): Unit = {
val locatedClass = {
JarUtils.outputJar match {
case Some(outputJar) => locateClassInJar(sym, outputJar, separatorRequired)
case None => locatePlainClassFile(sym, outputDir, separatorRequired)
}
}

locatedClass.foreach { classFile =>
assert(sym.isClass, s"${sym.fullName} is not a class")
// Use own map of local classes computed before lambdalift to ascertain class locality
if (localToNonLocalClass.isLocal(sym).getOrElse(true)) {
// Inform callback about local classes, non-local classes have been reported in API
callback.generatedLocalClass(sourceFile, classFile.toPath)
}
}
}

if (sym.isModuleClass && !sym.isImplClass) {
if (isTopLevelModule(sym) && sym.companionClass == NoSymbol)
addGenerated(false)
addGenerated(true)
} else
addGenerated(false)
}
}
}

private def locatePlainClassFile(
sym: Symbol,
outputDir: File,
separatorRequired: Boolean
): Option[File] = {
val classFile = fileForClass(outputDir, sym, separatorRequired)
if (classFile.exists()) Some(classFile) else None
}

private def locateClassInJar(sym: Symbol, jar: Path, sepRequired: Boolean): Option[File] = {
val classFile = pathToClassFile(sym, sepRequired)
val classInJar = JarUtils.classNameInJar(jar, classFile)
if (!classesWrittenByGenbcode.contains(classInJar)) None
else Some(new File(classInJar))
}
}
}
Loading

0 comments on commit b8cebcc

Please sign in to comment.