Port #6057 - Pipeline code gen and post-processing

compiler/src/dotty/tools/backend/jvm/BCodeHelpers.scala

@@ -51,10 +51,9 @@ trait BCodeHelpers extends BCodeIdiomatic {
   import int.{_, given}
   import DottyBackendInterface._
 
-  protected def backendUtils = genBCodePhase match {
-    case genBCode: GenBCode => genBCode.postProcessor.backendUtils
-    case _ => null
-  }
+  // We need to access GenBCode phase to get access to post-processor components.
+  // At this point it should always be initialized already.
+  protected lazy val backendUtils = genBCodePhase.asInstanceOf[GenBCode].postProcessor.backendUtils
 
   def ScalaATTRName: String = "Scala"
   def ScalaSignatureATTRName: String = "ScalaSig"

compiler/src/dotty/tools/backend/jvm/BCodeSkelBuilder.scala

@@ -37,7 +37,7 @@ trait BCodeSkelBuilder extends BCodeHelpers {
   import bTypes._
   import coreBTypes._
   import bCodeAsmCommon._
-  
+
   lazy val NativeAttr: Symbol = requiredClass[scala.native]
 
   /** The destination of a value generated by `genLoadTo`. */

compiler/src/dotty/tools/backend/jvm/BTypes.scala

@@ -860,7 +860,6 @@ abstract class BTypes { self =>
    * Just a named pair, used in CoreBTypes.asmBoxTo/asmUnboxTo.
    */
   /*final*/ case class MethodNameAndType(name: String, methodType: MethodBType)
-
 }
 
 object BTypes {
@@ -870,6 +869,4 @@ object BTypes {
    * But that would create overhead in a Collection[InternalName].
    */
   type InternalName = String
-
-
-}
+}

compiler/src/dotty/tools/backend/jvm/BTypesFromSymbols.scala

@@ -36,7 +36,7 @@ class BTypesFromSymbols[I <: DottyBackendInterface](val int: I, val frontendAcce
   }
   import coreBTypes._
 
-  @threadUnsafe protected lazy val classBTypeFromInternalNameMap = 
+  @threadUnsafe protected lazy val classBTypeFromInternalNameMap =
     collection.concurrent.TrieMap.empty[String, ClassBType]
 
   /**

compiler/src/dotty/tools/backend/jvm/BackendUtils.scala

@@ -36,7 +36,7 @@ class BackendUtils(val postProcessor: PostProcessor) {
     case "19" => asm.Opcodes.V19
     case "20" => asm.Opcodes.V20
   }
-  
+
   lazy val extraProc: Int = {
     import GenBCodeOps.addFlagIf
     val majorVersion: Int = (classfileVersion & 0xFF)
@@ -68,27 +68,27 @@ class BackendUtils(val postProcessor: PostProcessor) {
   }
 
   /*
-    * Add:
-    *
-    * private static Object $deserializeLambda$(SerializedLambda l) {
-    *   try return indy[scala.runtime.LambdaDeserialize.bootstrap, targetMethodGroup$0](l)
-    *   catch {
-    *     case i: IllegalArgumentException =>
-    *       try return indy[scala.runtime.LambdaDeserialize.bootstrap, targetMethodGroup$1](l)
-    *       catch {
-    *         case i: IllegalArgumentException =>
-    *           ...
-    *             return indy[scala.runtime.LambdaDeserialize.bootstrap, targetMethodGroup${NUM_GROUPS-1}](l)
-    *       }
-    *
-    * We use invokedynamic here to enable caching within the deserializer without needing to
-    * host a static field in the enclosing class. This allows us to add this method to interfaces
-    * that define lambdas in default methods.
-    *
-    * SI-10232 we can't pass arbitrary number of method handles to the final varargs parameter of the bootstrap
-    * method due to a limitation in the JVM. Instead, we emit a separate invokedynamic bytecode for each group of target
-    * methods.
-    */
+  * Add:
+  *
+  * private static Object $deserializeLambda$(SerializedLambda l) {
+  *   try return indy[scala.runtime.LambdaDeserialize.bootstrap, targetMethodGroup$0](l)
+  *   catch {
+  *     case i: IllegalArgumentException =>
+  *       try return indy[scala.runtime.LambdaDeserialize.bootstrap, targetMethodGroup$1](l)
+  *       catch {
+  *         case i: IllegalArgumentException =>
+  *           ...
+  *             return indy[scala.runtime.LambdaDeserialize.bootstrap, targetMethodGroup${NUM_GROUPS-1}](l)
+  *       }
+  *
+  * We use invokedynamic here to enable caching within the deserializer without needing to
+  * host a static field in the enclosing class. This allows us to add this method to interfaces
+  * that define lambdas in default methods.
+  *
+  * SI-10232 we can't pass arbitrary number of method handles to the final varargs parameter of the bootstrap
+  * method due to a limitation in the JVM. Instead, we emit a separate invokedynamic bytecode for each group of target
+  * methods.
+  */
   def addLambdaDeserialize(classNode: ClassNode, implMethodsArray: Array[Handle]): Unit = {
     import asm.Opcodes._
     import bTypes._
@@ -161,7 +161,7 @@ class BackendUtils(val postProcessor: PostProcessor) {
    * `refedInnerClasses` may contain duplicates, need not contain the enclosing inner classes of
    * each inner class it lists (those are looked up and included).
    *
-   * This method serializes in the InnerClasses JVM attribute in an appropriate order, 
+   * This method serializes in the InnerClasses JVM attribute in an appropriate order,
    * not necessarily that given by `refedInnerClasses`.
    *
    * can-multi-thread

compiler/src/dotty/tools/backend/jvm/ClassfileWriter.scala

@@ -7,6 +7,7 @@ import java.util.jar.Attributes.Name
 import scala.tools.asm.ClassReader
 import scala.tools.asm.tree.ClassNode
 import dotty.tools.io.*
+import dotty.tools.dotc.core.Decorators.*
 import dotty.tools.dotc.util.NoSourcePosition
 import java.nio.charset.StandardCharsets
 import java.nio.channels.ClosedByInterruptException
@@ -15,36 +16,39 @@ import scala.language.unsafeNulls
 
 class ClassfileWriter(frontendAccess: PostProcessorFrontendAccess) {
   import frontendAccess.{backendReporting, compilerSettings}
-  
+
   // if non-null, classfiles are additionally written to this directory
   private val dumpOutputDir: AbstractFile = getDirectoryOrNull(compilerSettings.dumpClassesDirectory)
 
   // if non-null, classfiles are written to a jar instead of the output directory
-  private val jarWriter: JarWriter = 
-    val f = compilerSettings.outputDirectory
-    if f.hasExtension("jar") then
-      // If no main class was specified, see if there's only one
-      // entry point among the classes going into the jar.
-      val mainClass = compilerSettings.mainClass match {
-        case c @ Some(m) => 
-          backendReporting.log(s"Main-Class was specified: ${m}")
-          c
-        case None => frontendAccess.getEntryPoints match {
-          case Nil      =>
-            backendReporting.log("No Main-Class designated or discovered.")
-            None
+  private val jarWriter: JarWriter | Null = compilerSettings.outputDirectory match {
+    case jar: JarArchive =>
+      val mainClass = compilerSettings.mainClass.orElse {
+        // If no main class was specified, see if there's only one
+        // entry point among the classes going into the jar.
+        frontendAccess.getEntryPoints match {
           case name :: Nil =>
-            backendReporting.log(s"Unique entry point: setting Main-Class to $name")
+            backendReporting.log(i"Unique entry point: setting Main-Class to $name")
             Some(name)
           case names =>
-            backendReporting.log(s"No Main-Class due to multiple entry points:\n  ${names.mkString("\n  ")}")
+            if names.isEmpty then backendReporting.warning(em"No Main-Class designated or discovered.")
+            else backendReporting.warning(em"No Main-Class due to multiple entry points:\n  ${names.mkString("\n  ")}")
             None
         }
       }
+      jar.underlyingSource.map{ source =>
+        if jar.isEmpty then
+          val jarMainAttrs = mainClass.map(Name.MAIN_CLASS -> _).toList
+          new Jar(source.file).jarWriter(jarMainAttrs: _*)
+        else
+          // Writing to non-empty JAR might be an undefined behaviour, e.g. in case if other files where
+          // created using `AbstractFile.bufferedOutputStream`instead of JarWritter
+          backendReporting.warning(em"Tried to write to non-empty JAR: $source")
+          null
+      }.orNull
 
-      val jarMainAttrs = mainClass.map(c => Name.MAIN_CLASS -> c).toList
-      new Jar(f.file).jarWriter(jarMainAttrs: _*)
-    else null
+    case _ => null
+  }
 
   private def getDirectoryOrNull(dir: Option[String]): AbstractFile =
     dir.map(d => new PlainDirectory(Directory(d))).orNull
@@ -88,20 +92,9 @@ class ClassfileWriter(frontendAccess: PostProcessorFrontendAccess) {
     }
   }
 
-  def write(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile | Null = try {
+  def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile | Null = try {
     // val writeStart = Statistics.startTimer(BackendStats.bcodeWriteTimer)
-    val outFile = if (jarWriter == null) {
-      val outFolder = compilerSettings.outputDirectory
-      val outFile = getFile(outFolder, className, ".class")
-      writeBytes(outFile, bytes)
-      outFile
-    } else {
-      val path = className + ".class"
-      val out = jarWriter.newOutputStream(path)
-      try out.write(bytes, 0, bytes.length)
-      finally out.flush()
-      null
-    }
+    val outFile = writeToJarOrFile(className, bytes, ".class")
     // Statistics.stopTimer(BackendStats.bcodeWriteTimer, writeStart)
 
     if (dumpOutputDir != null) {
@@ -111,27 +104,39 @@ class ClassfileWriter(frontendAccess: PostProcessorFrontendAccess) {
     outFile
   } catch {
     case e: FileConflictException =>
-      backendReporting.error(s"error writing $className: ${e.getMessage}", NoSourcePosition)
+      backendReporting.error(em"error writing $className: ${e.getMessage}")
       null
     case e: java.nio.file.FileSystemException =>
       if compilerSettings.debug then e.printStackTrace()
-      backendReporting.error(s"error writing $className: ${e.getClass.getName} ${e.getMessage}", NoSourcePosition)
+      backendReporting.error(em"error writing $className: ${e.getClass.getName} ${e.getMessage}")
       null
   }
 
-  def writeTasty(className: InternalName, bytes: Array[Byte]): Unit = 
-    val outFolder = compilerSettings.outputDirectory
-    val outFile = getFile(outFolder, className, ".tasty")
-    try writeBytes(outFile, bytes)
-    catch case ex: ClosedByInterruptException =>
-      try outFile.delete() // don't leave an empty or half-written tastyfile around after an interrupt
-      catch case _: Throwable => ()
-      finally throw ex
+  def writeTasty(className: InternalName, bytes: Array[Byte]): Unit =
+    writeToJarOrFile(className, bytes, ".tasty")
+
+  private def writeToJarOrFile(className: InternalName, bytes: Array[Byte], suffix: String): AbstractFile | Null = {
+    if jarWriter == null then
+      val outFolder = compilerSettings.outputDirectory
+      val outFile = getFile(outFolder, className, suffix)
+      try writeBytes(outFile, bytes)
+      catch case ex: ClosedByInterruptException =>
+        try outFile.delete() // don't leave an empty or half-written files around after an interrupt
+        catch case _: Throwable => ()
+        finally throw ex
+      outFile
+    else
+      val path = className + suffix
+      val out = jarWriter.newOutputStream(path)
+      try out.write(bytes, 0, bytes.length)
+      finally out.flush()
+      null
+  }
 
   def close(): Unit = {
     if (jarWriter != null) jarWriter.close()
   }
 }
 
 /** Can't output a file due to the state of the file system. */
-class FileConflictException(msg: String, val file: AbstractFile) extends IOException(msg)
+class FileConflictException(msg: String, val file: AbstractFile) extends IOException(msg)

compiler/src/dotty/tools/backend/jvm/CodeGen.scala

@@ -40,51 +40,72 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
 
   private lazy val mirrorCodeGen = Impl.JMirrorBuilder()
 
-  def gen(unit: CompilationUnit): Unit = {
-    val postProcessor = Phases.genBCodePhase match {
-      case genBCode: GenBCode => genBCode.postProcessor
-      case _ => null
-    }
+  def genUnit(unit: CompilationUnit): GeneratedDefs = {
+    val generatedClasses = mutable.ListBuffer.empty[GeneratedClass]
+    val generatedTasty = mutable.ListBuffer.empty[GeneratedTasty]
 
     def genClassDef(cd: TypeDef): Unit =
       try
         val sym = cd.symbol
         val sourceFile = unit.source.file
 
-        def registerGeneratedClass(classNode: ClassNode, isArtifact: Boolean) = if (classNode != null){
-          postProcessor.generatedClasses += GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated(classNode, sym, unit.source))
-        }
+        def registerGeneratedClass(classNode: ClassNode, isArtifact: Boolean): Unit =
+          generatedClasses += GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated(classNode, sym, unit.source))
 
         val plainC = genClass(cd, unit)
         registerGeneratedClass(plainC, isArtifact = false)
 
-        val mirrorC =
-          if !sym.isTopLevelModuleClass then null
-          else if sym.companionClass == NoSymbol then genMirrorClass(sym, unit)
+        val attrNode =
+          if !sym.isTopLevelModuleClass then plainC
+          else if sym.companionClass == NoSymbol then
+            val mirrorC = genMirrorClass(sym, unit)
+            registerGeneratedClass(mirrorC, isArtifact = true)
+            mirrorC
           else
             report.log(s"No mirror class for module with linked class: ${sym.fullName}", NoSourcePosition)
-            null
-        registerGeneratedClass(mirrorC, isArtifact = true)
+            plainC
 
         if sym.isClass then
-          val attrNode = if (mirrorC ne null) mirrorC else plainC
-          setTastyAttributes(sym, attrNode, postProcessor, unit)
+          genTastyAndSetAttributes(sym, attrNode)
       catch
         case ex: Throwable =>
           ex.printStackTrace()
           report.error(s"Error while emitting ${unit.source}\n${ex.getMessage}", NoSourcePosition)
 
+
+    def genTastyAndSetAttributes(claszSymbol: Symbol, store: ClassNode): Unit =
+      import Impl.createJAttribute
+      for (binary <- unit.pickled.get(claszSymbol.asClass)) {
+        generatedTasty += GeneratedTasty(store, binary)
+        val tasty =
+          val uuid = new TastyHeaderUnpickler(binary()).readHeader()
+          val lo = uuid.getMostSignificantBits
+          val hi = uuid.getLeastSignificantBits
+
+          // TASTY attribute is created but only the UUID bytes are stored in it.
+          // A TASTY attribute has length 16 if and only if the .tasty file exists.
+          val buffer = new TastyBuffer(16)
+          buffer.writeUncompressedLong(lo)
+          buffer.writeUncompressedLong(hi)
+          buffer.bytes
+
+        val dataAttr = createJAttribute(nme.TASTYATTR.mangledString, tasty, 0, tasty.length)
+        store.visitAttribute(dataAttr)
+      }
+
     def genClassDefs(tree: Tree): Unit =
       tree match {
         case EmptyTree => ()
         case PackageDef(_, stats) => stats foreach genClassDefs
-        case ValDef(_,_,_) => () // module val not emmited
+        case ValDef(_, _, _) => () // module val not emitted
         case td: TypeDef =>  genClassDef(td)
       }
 
     genClassDefs(unit.tpdTree)
+    GeneratedDefs(generatedClasses.toList, generatedTasty.toList)
   }
 
+  // Creates a callback that will be evaluated in PostProcessor after creating a file
   private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: interfaces.SourceFile): AbstractFile => Unit = clsFile => {
     val (fullClassName, isLocal) = atPhase(sbtExtractDependenciesPhase) {
       (ExtractDependencies.classNameAsString(claszSymbol), claszSymbol.isLocal)
@@ -112,28 +133,8 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
       override def jfile = Optional.ofNullable(absfile.file)
     }
 
-  private def setTastyAttributes(claszSymbol: Symbol, store: ClassNode, postProcessor: PostProcessor, unit: CompilationUnit): Unit =
-    import Impl.createJAttribute
-    for (binary <- unit.pickled.get(claszSymbol.asClass)) {
-      postProcessor.generatedTasty += GeneratedTasty(store, binary)
-      val tasty =
-        val uuid = new TastyHeaderUnpickler(binary()).readHeader()
-        val lo = uuid.getMostSignificantBits
-        val hi = uuid.getLeastSignificantBits
-
-        // TASTY attribute is created but only the UUID bytes are stored in it.
-        // A TASTY attribute has length 16 if and only if the .tasty file exists.
-        val buffer = new TastyBuffer(16)
-        buffer.writeUncompressedLong(lo)
-        buffer.writeUncompressedLong(hi)
-        buffer.bytes
-
-      val dataAttr = createJAttribute(nme.TASTYATTR.mangledString, tasty, 0, tasty.length)
-      store.visitAttribute(dataAttr)
-    }
-
   private def genClass(cd: TypeDef, unit: CompilationUnit): ClassNode = {
-    val b = new Impl.SyncAndTryBuilder(unit) {}
+    val b = new Impl.PlainClassBuilder(unit)
     b.genPlainClass(cd)
     val cls = b.cnode
     checkForCaseConflict(cls.name, cd.symbol)
@@ -174,5 +175,7 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
     val bTypes: self.bTypes.type = self.bTypes
     protected val primitives: DottyPrimitives = self.primitives
   }
-  object Impl extends ImplEarlyInit with BCodeSyncAndTry
-}
+  object Impl extends ImplEarlyInit with BCodeSyncAndTry {
+    class PlainClassBuilder(unit: CompilationUnit) extends SyncAndTryBuilder(unit)
+  }
+}