Skip to content

Commit

Permalink
Fuse emitting and printing of trees in the backend
Browse files Browse the repository at this point in the history 
This allows us to use the Emitter's powerful caching mechanism to
directly cache printed trees (as byte buffers) and not cache
JavaScript trees anymore at all.

This reduces in-between run memory usage on the test suite from
1.12 GB (not GiB) to 1.00 GB on my machine (roughly 10%).

Runtime performance (both batch and incremental is unaffected).

It is worth pointing out, that due to how the Emitter caches trees,
classes that end up being ES6 classes is performed will be held twice
in memory (once the individual methods, once the entire class).

On the test suite, this is the case for 710 cases out of 6538.
  • Loading branch information
@gzm0
gzm0 committed Dec 9, 2023
1 parent 6af57c9 commit 82a25db
Show file tree
Hide file tree
Showing 10 changed files with 419 additions and 301 deletions.
138 changes: 74 additions & 64 deletions linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureAstTransformer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import org.scalajs.ir
import ir.Position
import ir.Position.NoPosition

import org.scalajs.linker.backend.emitter.Emitter
import org.scalajs.linker.backend.javascript.Trees._
import org.scalajs.linker.backend.javascript.SourceFileUtil

Expand All @@ -30,30 +31,65 @@ import java.lang.{Double => JDouble}
import java.net.URI

private[closure] object ClosureAstTransformer {
def transformScript(topLevelTrees: List[Tree], featureSet: FeatureSet,
relativizeBaseURI: Option[URI]): Node = {
val transformer = new ClosureAstTransformer(featureSet, relativizeBaseURI)
transformer.transformScript(topLevelTrees)
}
val emptyChunk = new Chunk(Nil)

final class Chunk private[ClosureAstTransformer] (
private[ClosureAstTransformer] val nodes: List[Node]
) extends Transformed.Value
}

private class ClosureAstTransformer(featureSet: FeatureSet,
relativizeBaseURI: Option[URI]) {
private[closure] class ClosureAstTransformer(featureSet: FeatureSet,
relativizeBaseURI: Option[URI]) extends Emitter.PostTransformer[ClosureAstTransformer.Chunk] {
import ClosureAstTransformer.Chunk

private val dummySourceName = new java.net.URI("virtualfile:scala.js-ir")

def transformScript(topLevelTrees: List[Tree]): Node = {
def transformScript(chunks: List[Chunk]): Node = {
val script = setNodePosition(new Node(Token.SCRIPT), NoPosition)
transformBlockStats(topLevelTrees)(NoPosition).foreach(script.addChildToBack(_))

for {
chunk <- chunks
node <- chunk.nodes
} {
script.addChildToBack(node)
}

script.putProp(Node.FEATURE_SET, featureSet)
script
}

def transformStat(tree: Tree)(implicit parentPos: Position): Node =
def empty: Chunk = ClosureAstTransformer.emptyChunk

def transform(trees: List[Tree]): Chunk =
new Chunk(transformBlockStats(trees)(NoPosition))

private def transformStat(tree: Tree)(implicit parentPos: Position): Node =
innerTransformStat(tree, tree.pos orElse parentPos)

private def innerTransformStat(tree: Tree, pos_in: Position): Node = {
implicit val pos = pos_in

def newFixedPropNode(token: Token, static: Boolean, name: Ident,
function: Node): Node = {
val node = Node.newString(token, name.name)
node.addChildToBack(function)
node.setStaticMember(static)
node
}

/* This method should take a `prop: Node.Prop` parameter to factor out
* the `node.putBooleanProp()` that we find the three cases below. However,
* that is not possible because `Node.Prop` is private in `Node`. Go figure
* why Java allows to export as `public` the aliases
* `Node.COMPUTED_PROP_METHOD` et al. with a type that is not public ...
*/
def newComputedPropNode(static: Boolean, nameExpr: Tree,
function: Node): Node = {
val node = new Node(Token.COMPUTED_PROP, transformExpr(nameExpr), function)
node.setStaticMember(static)
node
}

wrapTransform(tree) {
case VarDef(ident, optRhs) =>
val node = transformName(ident)
Expand Down Expand Up @@ -172,51 +208,6 @@ private class ClosureAstTransformer(featureSet: FeatureSet,
case classDef: ClassDef =>
transformClassDef(classDef)

case _ =>
// We just assume it is an expression
new Node(Token.EXPR_RESULT, transformExpr(tree))
}
}

private def transformClassDef(classDef: ClassDef)(
implicit pos: Position): Node = {
val ClassDef(className, parentClass, members) = classDef

val membersBlock = new Node(Token.CLASS_MEMBERS)
for (member <- members)
membersBlock.addChildToBack(transformClassMember(member))
new Node(
Token.CLASS,
className.fold(new Node(Token.EMPTY))(transformName(_)),
parentClass.fold(new Node(Token.EMPTY))(transformExpr(_)),
membersBlock)
}

private def transformClassMember(member: Tree): Node = {
implicit val pos = member.pos

def newFixedPropNode(token: Token, static: Boolean, name: Ident,
function: Node): Node = {
val node = Node.newString(token, name.name)
node.addChildToBack(function)
node.setStaticMember(static)
node
}

/* This method should take a `prop: Node.Prop` parameter to factor out
* the `node.putBooleanProp()` that we find the three cases below. However,
* that is not possible because `Node.Prop` is private in `Node`. Go figure
* why Java allows to export as `public` the aliases
* `Node.COMPUTED_PROP_METHOD` et al. with a type that is not public ...
*/
def newComputedPropNode(static: Boolean, nameExpr: Tree,
function: Node): Node = {
val node = new Node(Token.COMPUTED_PROP, transformExpr(nameExpr), function)
node.setStaticMember(static)
node
}

wrapTransform(member) {
case MethodDef(static, name, args, restParam, body) =>
val function = genFunction("", args, restParam, body)
name match {
Expand Down Expand Up @@ -280,12 +271,27 @@ private class ClosureAstTransformer(featureSet: FeatureSet,
}

case _ =>
throw new AssertionError(
s"Unexpected class member tree of class ${member.getClass.getName}")
// We just assume it is an expression
new Node(Token.EXPR_RESULT, transformExpr(tree))
}
}

def transformExpr(tree: Tree)(implicit parentPos: Position): Node =
private def transformClassDef(classDef: ClassDef)(
implicit pos: Position): Node = {
val ClassDef(className, parentClass, members) = classDef

val membersBlock = new Node(Token.CLASS_MEMBERS)
for (node <- transformBlockStats(members))
membersBlock.addChildToBack(node)

new Node(
Token.CLASS,
className.fold(new Node(Token.EMPTY))(transformName(_)),
parentClass.fold(new Node(Token.EMPTY))(transformExpr(_)),
membersBlock)
}

private def transformExpr(tree: Tree)(implicit parentPos: Position): Node =
innerTransformExpr(tree, tree.pos orElse parentPos)

private def innerTransformExpr(tree: Tree, pos_in: Position): Node = {
Expand Down Expand Up @@ -406,15 +412,15 @@ private class ClosureAstTransformer(featureSet: FeatureSet,
new Node(Token.FUNCTION, nameNode, paramList, transformBlock(body))
}

def transformName(ident: Ident)(implicit parentPos: Position): Node =
private def transformName(ident: Ident)(implicit parentPos: Position): Node =
setNodePosition(Node.newString(Token.NAME, ident.name),
ident.pos orElse parentPos)

def transformLabel(ident: Ident)(implicit parentPos: Position): Node =
private def transformLabel(ident: Ident)(implicit parentPos: Position): Node =
setNodePosition(Node.newString(Token.LABEL_NAME, ident.name),
ident.pos orElse parentPos)

def transformObjectLitField(name: PropertyName, value: Tree)(
private def transformObjectLitField(name: PropertyName, value: Tree)(
implicit parentPos: Position): Node = {

val transformedValue = transformExpr(value)
Expand All @@ -436,7 +442,7 @@ private class ClosureAstTransformer(featureSet: FeatureSet,
setNodePosition(node, name.pos.orElse(parentPos))
}

def transformBlock(tree: Tree)(implicit parentPos: Position): Node = {
private def transformBlock(tree: Tree)(implicit parentPos: Position): Node = {
val pos = if (tree.pos.isDefined) tree.pos else parentPos
wrapTransform(tree) {
case Block(stats) =>
Expand All @@ -446,14 +452,14 @@ private class ClosureAstTransformer(featureSet: FeatureSet,
} (pos)
}

def transformBlock(stats: List[Tree], blockPos: Position): Node = {
private def transformBlock(stats: List[Tree], blockPos: Position): Node = {
val block = new Node(Token.BLOCK)
for (node <- transformBlockStats(stats)(blockPos))
block.addChildToBack(node)
block
}

def transformBlockStats(stats: List[Tree])(
private def transformBlockStats(stats: List[Tree])(
implicit parentPos: Position): List[Node] = {

@inline def ctorDoc(): JSDocInfo = {
Expand All @@ -469,6 +475,10 @@ private class ClosureAstTransformer(featureSet: FeatureSet,
case DocComment(text) :: tss =>
loop(tss, nextIsCtor = text.startsWith("@constructor"), acc)

case Transformed(chunk: Chunk) :: tss =>
assert(!nextIsCtor)
loop(tss, nextIsCtor = false, chunk.nodes reverse_::: acc)

case t :: tss =>
val node = transformStat(t)
if (nextIsCtor) {
Expand Down
36 changes: 19 additions & 17 deletions linker/jvm/src/main/scala/org/scalajs/linker/backend/closure/ClosureLinkerBackend.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,20 +53,6 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config)
require(moduleKind != ModuleKind.ESModule,
s"Cannot use module kind $moduleKind with the Closure Compiler")

private[this] val emitter = {
val emitterConfig = Emitter.Config(config.commonConfig.coreSpec)
.withJSHeader(config.jsHeader)
.withOptimizeBracketSelects(false)
.withTrackAllGlobalRefs(true)
.withInternalModulePattern(m => OutputPatternsImpl.moduleName(config.outputPatterns, m.id))

new Emitter(emitterConfig)
}

val symbolRequirements: SymbolRequirement = emitter.symbolRequirements

override def injectedIRFiles: Seq[IRFile] = emitter.injectedIRFiles

private val languageMode: ClosureOptions.LanguageMode = {
import ClosureOptions.LanguageMode._

Expand All @@ -85,6 +71,23 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config)
}
}

private[this] val transformer = new ClosureAstTransformer(
languageMode.toFeatureSet(), config.relativizeSourceMapBase)

private[this] val emitter: Emitter[ClosureAstTransformer.Chunk] = {
val emitterConfig = Emitter.Config(config.commonConfig.coreSpec)
.withJSHeader(config.jsHeader)
.withOptimizeBracketSelects(false)
.withTrackAllGlobalRefs(true)
.withInternalModulePattern(m => OutputPatternsImpl.moduleName(config.outputPatterns, m.id))

new Emitter(emitterConfig, transformer)
}

val symbolRequirements: SymbolRequirement = emitter.symbolRequirements

override def injectedIRFiles: Seq[IRFile] = emitter.injectedIRFiles

/** Emit the given [[standard.ModuleSet ModuleSet]] to the target output.
*
* @param moduleSet [[standard.ModuleSet ModuleSet]] to emit
Expand Down Expand Up @@ -129,9 +132,8 @@ final class ClosureLinkerBackend(config: LinkerBackendImpl.Config)
}
}

private def buildChunk(topLevelTrees: List[js.Tree]): JSChunk = {
val root = ClosureAstTransformer.transformScript(topLevelTrees,
languageMode.toFeatureSet(), config.relativizeSourceMapBase)
private def buildChunk(topLevelTrees: List[ClosureAstTransformer.Chunk]): JSChunk = {
val root = transformer.transformScript(topLevelTrees)

val chunk = new JSChunk("Scala.js")
chunk.add(new CompilerInput(new SyntheticAst(root)))
Expand Down

0 comments on commit 82a25db

Please sign in to comment.