Skip to content

Commit

Permalink
Add scala2-library-cc-tasty project (#18967)
Browse files Browse the repository at this point in the history
We can override the sources of the standard library with cc annotation
in this project and generate a TASTy version of the standard library.

Usage:
```
sbt> set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty
sbt> scala3-bootstrapped/testCompilation
sbt> scala3-bootstrapped/test
sbt> scala3-bootstrapped/run ...
```

```
sbt> scala2-library-cc/run diff
sbt> scala2-library-cc/run diff scala/Option.scala
sbt> scala2-library-cc/run diff scala/collection/immutable
```

### Current limitations

#### Empty self types in library
We need to override `Function1` and `PartialFunction` to remove their
empty self type. These cause some problem with capture checking. We need
to fix these cases. I will open an issue once we have a way to reproduce
it with this PR.

#### Cannot use the TASTy jar yet
There is a bug when we try to class load the captured checked library
from TASTy. These are due to some cyclic reference while unpickling the
TASTy.

<del>The current theory is that this due to the CC annotations in the
top-level classes.
If this is the case, we can fix this issue by adding these language
features as TASTy attributes (see #19033).</del>
This was wrong or only part of the problems. The current issue can be
reproduced with #19115.
  • Loading branch information
nicolasstucki committed Jan 3, 2024
2 parents 938d405 + 5c6e542 commit 25eb48e
Show file tree
Hide file tree
Showing 158 changed files with 914 additions and 287 deletions.
11 changes: 9 additions & 2 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,10 @@ jobs:
./project/scripts/sbt ";sjsSandbox/run ;sjsSandbox/test ;sjsJUnitTests/test ;set sjsJUnitTests/scalaJSLinkerConfig ~= switchToESModules ;sjsJUnitTests/test ;sjsCompilerTests/test"
- name: Test with Scala 2 library TASTy (fast)
run: ./project/scripts/sbt ";set ThisBuild/Build.useScala2LibraryTasty := true ;scala3-bootstrapped/testCompilation i5; scala3-bootstrapped/testCompilation tests/run/typelevel-peano.scala; scala3-bootstrapped/testOnly dotty.tools.backend.jvm.DottyBytecodeTests" # only test a subset of test to avoid doubling the CI execution time
run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryTasty ;scala3-bootstrapped/testCompilation i5; scala3-bootstrapped/testCompilation tests/run/typelevel-peano.scala; scala3-bootstrapped/testOnly dotty.tools.backend.jvm.DottyBytecodeTests" # only test a subset of test to avoid doubling the CI execution time

- name: Test with Scala 2 library with CC TASTy (fast)
run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty ;scala2-library-tasty/compile" # TODO test all the test configurations in non-CC library (currently disabled due to bug while loading the library)

test_scala2_library_tasty:
runs-on: [self-hosted, Linux]
Expand Down Expand Up @@ -186,7 +189,11 @@ jobs:
run: cp -vf .github/workflows/repositories /root/.sbt/ ; true

- name: Test with Scala 2 library TASTy
run: ./project/scripts/sbt ";set ThisBuild/Build.useScala2LibraryTasty := true ;scala3-bootstrapped/test"
run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryTasty ;scala3-bootstrapped/test"

# TODO test all the test configurations in non-CC library (currently disabled due to bug while loading the library)
# - name: Test with Scala 2 library with CC TASTy
# run: ./project/scripts/sbt ";set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty ;scala3-bootstrapped/test"


test_windows_fast:
Expand Down
2 changes: 2 additions & 0 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ val `scala3-bench-micro` = Build.`scala3-bench-micro`
val `scala2-library-bootstrapped` = Build.`scala2-library-bootstrapped`
val `scala2-library-tasty` = Build.`scala2-library-tasty`
val `scala2-library-tasty-tests` = Build.`scala2-library-tasty-tests`
val `scala2-library-cc` = Build.`scala2-library-cc`
val `scala2-library-cc-tasty` = Build.`scala2-library-cc-tasty`
val `tasty-core` = Build.`tasty-core`
val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped`
val `tasty-core-scala2` = Build.`tasty-core-scala2`
Expand Down
3 changes: 3 additions & 0 deletions compiler/test/dotty/Properties.scala
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ object Properties {

/** If we are using the scala-library TASTy jar */
def usingScalaLibraryTasty: Boolean = scalaLibraryTasty.isDefined
/** If we are using the scala-library TASTy jar */

def usingScalaLibraryCCTasty: Boolean = scalaLibraryTasty.exists(_.contains("scala2-library-cc-tasty"))

/** scala-asm jar */
def scalaAsm: String = sys.props("dotty.tests.classes.scalaAsm")
Expand Down
6 changes: 3 additions & 3 deletions compiler/test/dotty/tools/dotc/CompilationTests.scala
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ class CompilationTests {
compileFilesInDir("tests/pos", defaultOptions.and("-Ysafe-init", "-Ylegacy-lazy-vals", "-Ycheck-constraint-deps"), FileFilter.include(TestSources.posLazyValsAllowlist)),
compileDir("tests/pos-special/java-param-names", defaultOptions.withJavacOnlyOptions("-parameters")),
) ::: (
// FIXME: This fails due to a bug involving self types and capture checking
if Properties.usingScalaLibraryTasty then Nil
else List(compileDir("tests/pos-special/stdlib", allowDeepSubtypes))
// TODO create a folder for capture checking tests with the stdlib, or use tests/pos-custom-args/captures under this mode?
if Properties.usingScalaLibraryCCTasty then List(compileDir("tests/pos-special/stdlib", allowDeepSubtypes))
else Nil
)

if scala.util.Properties.isJavaAtLeast("16") then
Expand Down
98 changes: 77 additions & 21 deletions project/Build.scala
Original file line number Diff line number Diff line change
Expand Up @@ -173,11 +173,23 @@ object Build {
// Run tests with filter through vulpix test suite
val testCompilation = inputKey[Unit]("runs integration test with the supplied filter")

sealed trait Scala2Library
// Use Scala 2 compiled library JAR
object Scala2LibraryJar extends Scala2Library
// Use the TASTy jar from `scala2-library-tasty` in the classpath
// This only works with `scala3-bootstrapped/scalac` and tests in `scala3-bootstrapped`
//
// Enable in SBT with: `set ThisBuild/Build.useScala2LibraryTasty := true`
val useScala2LibraryTasty = settingKey[Boolean]("Use the TASTy jar from `scala2-library-tasty` in the classpath")
object Scala2LibraryTasty extends Scala2Library
// Use the TASTy jar from `scala2-library-cc-tasty` in the classpath
// This only works with `scala3-bootstrapped/scalac` and tests in `scala3-bootstrapped`
//
object Scala2LibraryCCTasty extends Scala2Library

// Set in SBT with:
// - `set ThisBuild/Build.scala2Library := Build.Scala2LibraryJar` (default)
// - `set ThisBuild/Build.scala2Library := Build.Scala2LibraryTasty`
// - `set ThisBuild/Build.scala2Library := Build.Scala2LibraryCCTasty`
val scala2Library = settingKey[Scala2Library]("Choose which version of the Scala 2 library should be used")

// Used to compile files similar to ./bin/scalac script
val scalac = inputKey[Unit]("run the compiler using the correct classpath, or the user supplied classpath")
Expand Down Expand Up @@ -225,7 +237,7 @@ object Build {

outputStrategy := Some(StdoutOutput),

useScala2LibraryTasty := false,
scala2Library := Scala2LibraryJar,

// enable verbose exception messages for JUnit
(Test / testOptions) += Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "-s"),
Expand Down Expand Up @@ -645,12 +657,18 @@ object Build {
val externalDeps = externalCompilerClasspathTask.value
val jars = packageAll.value

val scala2LibraryTasty = jars.get("scala2-library-tasty") match {
case Some(scala2LibraryTastyJar) if useScala2LibraryTasty.value =>
Seq("-Ddotty.tests.tasties.scalaLibrary=" + scala2LibraryTastyJar)
case _ =>
if (useScala2LibraryTasty.value) log.warn("useScala2LibraryTasty is ignored on non-bootstrapped compiler")
Seq.empty
def libraryPathProperty(jarName: String): Seq[String] =
jars.get(jarName) match {
case Some(jar) =>
Seq(s"-Ddotty.tests.tasties.scalaLibrary=$jar")
case None =>
log.warn("Scala 2 library TASTy is ignored on non-bootstrapped compiler")
Seq.empty
}
val scala2LibraryTasty = scala2Library.value match {
case Scala2LibraryJar => Seq.empty
case Scala2LibraryTasty => libraryPathProperty("scala2-library-tasty")
case Scala2LibraryCCTasty => libraryPathProperty("scala2-library-cc-tasty")
}

scala2LibraryTasty ++ Seq(
Expand Down Expand Up @@ -764,14 +782,24 @@ object Build {
else if (debugFromTasty) "dotty.tools.dotc.fromtasty.Debug"
else "dotty.tools.dotc.Main"

var extraClasspath =
scalaLibTastyOpt match {
case Some(scalaLibTasty) if useScala2LibraryTasty.value =>
Seq(scalaLibTasty, scalaLib, dottyLib)
case _ =>
if (useScala2LibraryTasty.value) log.warn("useScala2LibraryTasty is ignored on non-bootstrapped compiler")
Seq(scalaLib, dottyLib)
}
val scala2LibraryTasty = scala2Library.value match {
case Scala2LibraryJar => Seq.empty
case Scala2LibraryTasty =>
jars.get("scala2-library-tasty") match {
case Some(jar) => Seq(jar)
case None =>
log.warn("Scala2LibraryTasty is ignored on non-bootstrapped compiler")
Seq.empty
}
case Scala2LibraryCCTasty =>
jars.get("scala2-library-cc-tasty") match {
case Some(jar) => Seq(jar)
case None =>
log.warn("Scala2LibraryCCTasty is ignored on non-bootstrapped compiler")
Seq.empty
}
}
var extraClasspath = scala2LibraryTasty ++ Seq(scalaLib, dottyLib)

if (decompile && !args.contains("-classpath"))
extraClasspath ++= Seq(".")
Expand Down Expand Up @@ -882,6 +910,7 @@ object Build {
"scala3-tasty-inspector" -> (LocalProject("scala3-tasty-inspector") / Compile / packageBin).value.getAbsolutePath,
"tasty-core" -> (LocalProject("tasty-core-bootstrapped") / Compile / packageBin).value.getAbsolutePath,
"scala2-library-tasty" -> (LocalProject("scala2-library-tasty") / Compile / packageBin).value.getAbsolutePath,
"scala2-library-cc-tasty" -> (LocalProject("scala2-library-cc-tasty") / Compile / packageBin).value.getAbsolutePath,
)
},

Expand Down Expand Up @@ -1010,8 +1039,24 @@ object Build {
withCommonSettings(Bootstrapped).
dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
settings(commonBootstrappedSettings).
settings(scala2LibraryBootstrappedSettings).
settings(moduleName := "scala2-library")

/** Scala 2 library compiled by dotty using the latest published sources of the library.
*
* This version of the library is not (yet) TASTy/binary compatible with the Scala 2 compiled library.
*/
lazy val `scala2-library-cc` = project.in(file("scala2-library-cc")).
withCommonSettings(Bootstrapped).
dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
settings(commonBootstrappedSettings).
settings(scala2LibraryBootstrappedSettings).
settings(
moduleName := "scala2-library",
moduleName := "scala2-library-cc",
scalacOptions += "-Ycheck:all",
)

lazy val scala2LibraryBootstrappedSettings = Seq(
javaOptions := (`scala3-compiler-bootstrapped` / javaOptions).value,
Compile / scalacOptions ++= {
Seq("-sourcepath", ((Compile/sourceManaged).value / "scala-library-src").toString)
Expand Down Expand Up @@ -1096,13 +1141,13 @@ object Build {
| - final val MinorVersion = $minorVersion
| - final val ExperimentalVersion = 0
| * Clean everything to generate a compiler with those new TASTy versions
| * Run scala2-library-bootstrapped/tastyMiMaReportIssues
| * Run ${name.value}/tastyMiMaReportIssues
|""".stripMargin)

}).value,
Compile / exportJars := true,
artifactName := { (sv: ScalaVersion, module: ModuleID, artifact: Artifact) =>
"scala2-library-" + dottyVersion + "." + artifact.extension
moduleName.value + "-" + dottyVersion + "." + artifact.extension
},
run := {
val log = streams.value.log
Expand Down Expand Up @@ -1174,7 +1219,7 @@ object Build {
|""".stripMargin)
}
}
)
)

/** Packages the TASTy files of `scala2-library-bootstrapped` in a jar */
lazy val `scala2-library-tasty` = project.in(file("scala2-library-tasty")).
Expand All @@ -1187,6 +1232,17 @@ object Build {
},
)

/** Packages the TASTy files of `scala2-library-cc` in a jar */
lazy val `scala2-library-cc-tasty` = project.in(file("scala2-library-cc-tasty")).
withCommonSettings(Bootstrapped).
settings(
exportJars := true,
Compile / packageBin / mappings := {
(`scala2-library-cc` / Compile / packageBin / mappings).value
.filter(_._2.endsWith(".tasty"))
},
)

/** Test the tasty generated by `scala2-library-bootstrapped`
*
* The sources in src are compiled using TASTy from scala2-library-tasty but then run
Expand Down
90 changes: 90 additions & 0 deletions scala2-library-cc/src/scala/Function1.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Scala (https://www.scala-lang.org)
*
* Copyright EPFL and Lightbend, Inc.
*
* Licensed under Apache License 2.0
* (http://www.apache.org/licenses/LICENSE-2.0).
*
* See the NOTICE file distributed with this work for
* additional information regarding copyright ownership.
*/

// GENERATED CODE: DO NOT EDIT. See scala.Function0 for timestamp.

package scala


object Function1 {

implicit final class UnliftOps[A, B] private[Function1](private val f: A => Option[B]) extends AnyVal {
/** Converts an optional function to a partial function.
*
* @example Unlike [[Function.unlift]], this [[UnliftOps.unlift]] method can be used in extractors.
* {{{
* val of: Int => Option[String] = { i =>
* if (i == 2) {
* Some("matched by an optional function")
* } else {
* None
* }
* }
*
* util.Random.nextInt(4) match {
* case of.unlift(m) => // Convert an optional function to a pattern
* println(m)
* case _ =>
* println("Not matched")
* }
* }}}
*/
def unlift: PartialFunction[A, B] = Function.unlift(f)
}

}

/** A function of 1 parameter.
*
* In the following example, the definition of `succ` is
* shorthand, conceptually, for the anonymous class definition
* `anonfun1`, although the implementation details of how the
* function value is constructed may differ:
*
* {{{
* object Main extends App {
* val succ = (x: Int) => x + 1
* val anonfun1 = new Function1[Int, Int] {
* def apply(x: Int): Int = x + 1
* }
* assert(succ(0) == anonfun1(0))
* }
* }}}
*
* Note that the difference between `Function1` and [[scala.PartialFunction]]
* is that the latter can specify inputs which it will not handle.
*/
@annotation.implicitNotFound(msg = "No implicit view available from ${T1} => ${R}.")
trait Function1[@specialized(Specializable.Arg) -T1, @specialized(Specializable.Return) +R] extends AnyRef { // FIXME: self =>
/** Apply the body of this function to the argument.
* @return the result of function application.
*/
def apply(v1: T1): R

/** Composes two instances of Function1 in a new Function1, with this function applied last.
*
* @tparam A the type to which function `g` can be applied
* @param g a function A => T1
* @return a new function `f` such that `f(x) == apply(g(x))`
*/
@annotation.unspecialized def compose[A](g: A => T1): A => R = { x => apply(g(x)) }

/** Composes two instances of Function1 in a new Function1, with this function applied first.
*
* @tparam A the result type of function `g`
* @param g a function R => A
* @return a new function `f` such that `f(x) == g(apply(x))`
*/
@annotation.unspecialized def andThen[A](g: R => A): T1 => A = { x => g(apply(x)) }

override def toString(): String = "<function1>"
}
Loading

0 comments on commit 25eb48e

Please sign in to comment.