From 84cf02c249891d02b1eb38981b7df4ae9377e6e4 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Tue, 14 Mar 2017 15:07:21 +0100 Subject: [PATCH 1/4] Upgrade to sbt 0.13.14-RC2 Also replace com.typesafe.sbt:sbt-interface by the equivalent org.scala-sbt:interface since the former is not published in 0.13.14 for some reason (https://github.com/sbt/sbt/issues/3012). --- project/Build.scala | 13 ++++++++----- project/build.properties | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index f702e3563ac9..10a39767830a 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -281,8 +281,11 @@ object DottyBuild extends Build { libraryDependencies ++= partestDeps.value, libraryDependencies ++= Seq("org.scala-lang.modules" %% "scala-xml" % "1.0.1", "org.scala-lang.modules" %% "scala-partest" % "1.0.11" % "test", - "com.novocode" % "junit-interface" % "0.11" % "test", - "com.typesafe.sbt" % "sbt-interface" % sbtVersion.value), + "com.novocode" % "junit-interface" % "0.11" % "test"), + + resolvers += Resolver.typesafeIvyRepo("releases"), // For org.scala-sbt:interface + libraryDependencies += "org.scala-sbt" % "interface" % sbtVersion.value, + // enable improved incremental compilation algorithm incOptions := incOptions.value.withNameHashing(true), @@ -428,7 +431,7 @@ object DottyBuild extends Build { // FIXME: should go away when xml literal parsing is removed path.contains("scala-xml") || // needed for the xsbti interface - path.contains("sbt-interface") + path.contains("org.scala-sbt/interface/") } yield "-Xbootclasspath/p:" + path val ci_build = // propagate if this is a ci build @@ -552,9 +555,9 @@ object DottyBuild extends Build { }, publishLocal := (publishLocal.dependsOn(cleanSbtBridge)).value, description := "sbt compiler bridge for Dotty", - resolvers += Resolver.typesafeIvyRepo("releases"), + resolvers += Resolver.typesafeIvyRepo("releases"), // For org.scala-sbt stuff libraryDependencies ++= Seq( - "com.typesafe.sbt" % "sbt-interface" % sbtVersion.value, + "org.scala-sbt" % "interface" % sbtVersion.value, "org.scala-sbt" % "api" % sbtVersion.value % "test", "org.specs2" %% "specs2" % "2.3.11" % "test" ), diff --git a/project/build.properties b/project/build.properties index 43b8278c68cf..866eb2660546 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.11 +sbt.version=0.13.14-RC2 From 6099195216e36efff4ab7175881c6e32e52aae2f Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Tue, 14 Mar 2017 15:09:01 +0100 Subject: [PATCH 2/4] Upgrade sbt plugins, remove unused scalastyle plugin --- project/plugins.sbt | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 71a7ef5b6445..81d5ceb91af6 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,10 +3,8 @@ // e.g. addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.1.0") // Scala IDE project file generator -addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0") +addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.1.0") -addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.14") -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.8") - -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4") From d9f98c2848dbb0ae18ae75159761598fdac894c3 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Tue, 14 Mar 2017 18:52:23 +0100 Subject: [PATCH 3/4] sbt-bridge: Synchronize unit tests with sbt 0.13.14 Also fix a bug where the compiler output for the tests ended up in the wrong directory, causing some new tests from 0.13.14 to fail. --- .../test/xsbt/ExtractAPISpecification.scala | 80 ++++++++++++++++++- .../xsbt/ExtractUsedNamesSpecification.scala | 39 +++++++++ .../xsbt/ScalaCompilerForUnitTesting.scala | 47 +++++++---- sbt-bridge/test/xsbti/TestCallback.scala | 2 +- 4 files changed, 148 insertions(+), 20 deletions(-) diff --git a/sbt-bridge/test/xsbt/ExtractAPISpecification.scala b/sbt-bridge/test/xsbt/ExtractAPISpecification.scala index f5af67e454bd..4b3b2c51aaa9 100644 --- a/sbt-bridge/test/xsbt/ExtractAPISpecification.scala +++ b/sbt-bridge/test/xsbt/ExtractAPISpecification.scala @@ -2,9 +2,8 @@ package xsbt import org.junit.runner.RunWith -import xsbti.api.ClassLike -import xsbti.api.Def -import xsbt.api.ShowAPI +import xsbti.api._ +import xsbt.api.DefaultShowAPI import org.specs2.mutable.Specification import org.specs2.runner.JUnitRunner @@ -17,7 +16,7 @@ class ExtractAPISpecification extends Specification { def stableExistentialNames: Boolean = { def compileAndGetFooMethodApi(src: String): Def = { - val compilerForTesting = new ScalaCompilerForUnitTesting + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = false) val sourceApi = compilerForTesting.extractApiFromSrc(src) val FooApi = sourceApi.definitions().find(_.name() == "Foo").get.asInstanceOf[ClassLike] val fooMethodApi = FooApi.structure().declared().find(_.name == "foo").get @@ -38,8 +37,81 @@ class ExtractAPISpecification extends Specification { | }""".stripMargin val fooMethodApi2 = compileAndGetFooMethodApi(src2) + fooMethodApi1 == fooMethodApi2 // Fails because xsbt.api is compiled with Scala 2.10 // SameAPI.apply(fooMethodApi1, fooMethodApi2) } + + /** + * Checks if representation of the inherited Namer class (with a declared self variable) in Global.Foo + * is stable between compiling from source and unpickling. We compare extracted APIs of Global when Global + * is compiled together with Namers or Namers is compiled first and then Global refers + * to Namers by unpickling types from class files. + * + * See https://github.com/sbt/sbt/issues/2504 + */ + "Self variable and no self type" in { + def selectNamer(api: SourceAPI): ClassLike = { + def selectClass(defs: Iterable[Definition], name: String): ClassLike = defs.collectFirst { + case cls: ClassLike if cls.name == name => cls + }.get + val global = selectClass(api.definitions, "Global") + val foo = selectClass(global.structure.declared, "Global.Foo") + selectClass(foo.structure.inherited, "Namers.Namer") + } + val src1 = + """|class Namers { + | class Namer { thisNamer => } + |} + |""".stripMargin + val src2 = + """|class Global { + | class Foo extends Namers + |} + |""".stripMargin + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = false) + val apis = compilerForTesting.extractApisFromSrcs(reuseCompilerInstance = false)(List(src1, src2), List(src2)) + val _ :: src2Api1 :: src2Api2 :: Nil = apis.toList + val namerApi1 = selectNamer(src2Api1) + val namerApi2 = selectNamer(src2Api2) + + DefaultShowAPI(namerApi1) == DefaultShowAPI(namerApi2) + // Fails because xsbt.api is compiled with Scala 2.10 + // SameAPI(namerApi1, namerApi2) + } + + /** + * Checks if self type is properly extracted in various cases of declaring a self type + * with our without a self variable. + */ + "Self type" in { + def collectFirstClass(defs: Array[Definition]): ClassLike = defs.collectFirst { + case c: ClassLike => c + }.get + val srcX = "trait X" + val srcY = "trait Y" + val srcC1 = "class C1 { this: C1 => }" + val srcC2 = "class C2 { thisC: C2 => }" + val srcC3 = "class C3 { this: X => }" + val srcC4 = "class C4 { thisC: X => }" + val srcC5 = "class C5 extends AnyRef with X with Y { self: X with Y => }" + val srcC6 = "class C6 extends AnyRef with X { self: X with Y => }" + // val srcC7 = "class C7 { _ => }" // DOTTY: Syntax not supported + val srcC8 = "class C8 { self => }" + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = false) + val apis = compilerForTesting.extractApisFromSrcs(reuseCompilerInstance = true)( + List(srcX, srcY, srcC1, srcC2, srcC3, srcC4, srcC5, srcC6, srcC8) + ).map(x => collectFirstClass(x.definitions)) + val emptyType = new EmptyType + def hasSelfType(c: ClassLike): Boolean = + c.selfType != emptyType + val (withSelfType, withoutSelfType) = apis.partition(hasSelfType) + // DOTTY: In the scalac ExtractAPI phase, the self-type is only + // extracted if it differs from the type of the class for stability + // reasons. This isn't necessary in dotty because we always pickle + // the self type. + withSelfType.map(_.name).toSet === Set("C1", "C2", "C3", "C4", "C5", "C6", "C8") + withoutSelfType.map(_.name).toSet === Set("X", "Y") + } } diff --git a/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala b/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala index ed463a3e6525..6cff284feb15 100644 --- a/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala +++ b/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala @@ -75,6 +75,7 @@ class ExtractUsedNamesSpecification extends Specification { |}""".stripMargin val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) val usedNames = compilerForTesting.extractUsedNamesFromSrc(srcA, srcB) + // DOTTY TODO: "Int" is not actually used, but we collect it because // it's the inferred return type so it appears in a TypeTree // We could avoid this by checking if the untyped tree has a return type @@ -84,6 +85,44 @@ class ExtractUsedNamesSpecification extends Specification { usedNames === expectedNames } + "extract names in the types of trees" in { + val src1 = """|class X0 + |class X1 extends X0 + |class Y + |class A { + | type T >: X1 <: X0 + |} + |class M + |class N + |class P0 + |class P1 extends P0 + |object B { + | type S = Y + | val lista: List[A] = ??? + | val at: A#T = ??? + | val as: S = ??? + | def foo(m: M): N = ??? + | def bar[Param >: P1 <: P0](p: Param): Param = ??? + |}""".stripMargin + val src2 = """|object Test { + | val x = B.lista + | val y = B.at + | val z = B.as + | B.foo(???) + | B.bar(???) + |}""".stripMargin + val compilerForTesting = new ScalaCompilerForUnitTesting(nameHashing = true) + val usedNames = compilerForTesting.extractUsedNamesFromSrc(src1, src2) + val expectedNames = standardNames ++ Set("Test", "Test$", "B", "B$", + "Predef", "Predef$", "$qmark$qmark$qmark", "Nothing", + "lista", "List", "A", + "at", "T", "X1", "X0", + "as", "S", "Y", + "foo", "M", "N", + "bar", "P1", "P0") + usedNames === expectedNames + } + // test for https://github.com/gkossakowski/sbt/issues/3 "used names from the same compilation unit" in { val src = "class A { def foo: Int = 0; def bar: Int = foo }" diff --git a/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala b/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala index 40972902312c..a6b9fa65ede7 100644 --- a/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala +++ b/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala @@ -7,7 +7,7 @@ import _root_.scala.tools.nsc.reporters.ConsoleReporter import _root_.scala.tools.nsc.Settings import xsbti._ import xsbti.api.SourceAPI -import sbt.IO.withTemporaryDirectory +import sbt.IO._ import xsbti.api.ClassLike import xsbti.api.Definition import xsbti.api.Def @@ -21,7 +21,7 @@ import ScalaCompilerForUnitTesting.ExtractedSourceDependencies * Provides common functionality needed for unit tests that require compiling * source code using Scala compiler. */ -class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) { +class ScalaCompilerForUnitTesting(nameHashing: Boolean, includeSynthToNameHashing: Boolean = false) { import scala.language.reflectiveCalls /** @@ -33,6 +33,15 @@ class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) { analysisCallback.apis(tempSrcFile) } + /** + * Compiles given source code using Scala compiler and returns API representation + * extracted by ExtractAPI class. + */ + def extractApisFromSrcs(reuseCompilerInstance: Boolean)(srcs: List[String]*): Seq[SourceAPI] = { + val (tempSrcFiles, analysisCallback) = compileSrcs(srcs.toList, reuseCompilerInstance) + tempSrcFiles.map(analysisCallback.apis) + } + def extractUsedNamesFromSrc(src: String): Set[String] = { val (Seq(tempSrcFile), analysisCallback) = compileSrcs(src) analysisCallback.usedNames(tempSrcFile) @@ -66,7 +75,7 @@ class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) { def extractDependenciesFromSrcs(srcs: List[Map[Symbol, String]]): ExtractedSourceDependencies = { val rawGroupedSrcs = srcs.map(_.values.toList) val symbols = srcs.flatMap(_.keys) - val (tempSrcFiles, testCallback) = compileSrcs(rawGroupedSrcs) + val (tempSrcFiles, testCallback) = compileSrcs(rawGroupedSrcs, reuseCompilerInstance = true) val fileToSymbol = (tempSrcFiles zip symbols).toMap val memberRefFileDeps = testCallback.sourceDependencies collect { @@ -109,19 +118,31 @@ class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) { * useful to compile macros, which cannot be used in the same compilation run that * defines them. * + * The `reuseCompilerInstance` parameter controls whether the same Scala compiler instance + * is reused between compiling source groups. Separate compiler instances can be used to + * test stability of API representation (with respect to pickling) or to test handling of + * binary dependencies. + * * The sequence of temporary files corresponding to passed snippets and analysis * callback is returned as a result. */ - private def compileSrcs(groupedSrcs: List[List[String]]): (Seq[File], TestCallback) = { - withTemporaryDirectory { temp => - val analysisCallback = new TestCallback(nameHashing) + private def compileSrcs(groupedSrcs: List[List[String]], + reuseCompilerInstance: Boolean): (Seq[File], TestCallback) = { + // withTemporaryDirectory { temp => + { + val temp = createTemporaryDirectory + val analysisCallback = new TestCallback(nameHashing, includeSynthToNameHashing) val classesDir = new File(temp, "classes") classesDir.mkdir() - // val (compiler, ctx) = prepareCompiler(classesDir, analysisCallback, classesDir.toString) + lazy val commonCompilerInstanceAndCtx = prepareCompiler(classesDir, analysisCallback, classesDir.toString) val files = for ((compilationUnit, unitId) <- groupedSrcs.zipWithIndex) yield { - val (compiler, ctx) = prepareCompiler(classesDir, analysisCallback, classesDir.toString) + // use a separate instance of the compiler for each group of sources to + // have an ability to test for bugs in instability between source and pickled + // representation of types + val (compiler, ctx) = if (reuseCompilerInstance) commonCompilerInstanceAndCtx else + prepareCompiler(classesDir, analysisCallback, classesDir.toString) val run = compiler.newRun(ctx) val srcFiles = compilationUnit.toSeq.zipWithIndex map { case (src, i) => @@ -132,7 +153,7 @@ class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) { run.compile(srcFilePaths) - srcFilePaths.foreach(f => new File(f).delete) + // srcFilePaths.foreach(f => new File(f).delete) srcFiles } (files.flatten.toSeq, analysisCallback) @@ -140,7 +161,7 @@ class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) { } private def compileSrcs(srcs: String*): (Seq[File], TestCallback) = { - compileSrcs(List(srcs.toList)) + compileSrcs(List(srcs.toList), reuseCompilerInstance = true) } private def prepareSrcFile(baseDir: File, fileName: String, src: String): File = { @@ -151,10 +172,6 @@ class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) { private def prepareCompiler(outputDir: File, analysisCallback: AnalysisCallback, classpath: String = ".") = { val args = Array.empty[String] - object output extends SingleOutput { - def outputDirectory: File = outputDir - override def toString = s"SingleOutput($outputDirectory)" - } import dotty.tools.dotc._ import dotty.tools.dotc.core.Contexts._ @@ -171,7 +188,7 @@ class ScalaCompilerForUnitTesting(nameHashing: Boolean = false) { } } val ctx = (new ContextBase).initialCtx.fresh.setSbtCallback(analysisCallback) - driver.getCompiler(Array("-classpath", classpath, "-usejavacp"), ctx) + driver.getCompiler(Array("-classpath", classpath, "-usejavacp", "-d", outputDir.getAbsolutePath), ctx) } private object ConsoleReporter extends Reporter { diff --git a/sbt-bridge/test/xsbti/TestCallback.scala b/sbt-bridge/test/xsbti/TestCallback.scala index b849e1a8096c..99c8d963d555 100644 --- a/sbt-bridge/test/xsbti/TestCallback.scala +++ b/sbt-bridge/test/xsbti/TestCallback.scala @@ -6,7 +6,7 @@ import scala.collection.mutable.ArrayBuffer import xsbti.api.SourceAPI import xsbti.DependencyContext._ -class TestCallback(override val nameHashing: Boolean = false) extends AnalysisCallback +class TestCallback(override val nameHashing: Boolean, override val includeSynthToNameHashing: Boolean) extends AnalysisCallback { val sourceDependencies = new ArrayBuffer[(File, File, DependencyContext)] val binaryDependencies = new ArrayBuffer[(File, String, File, DependencyContext)] From 70c072abc729c1ffc080baff68dfd8fd6db1bcd1 Mon Sep 17 00:00:00 2001 From: Guillaume Martres Date: Fri, 24 Mar 2017 15:32:09 +0100 Subject: [PATCH 4/4] Switch to build.sbt Using the same technique than scala-js where we just forward to Build.scala: https://github.com/scala-js/scala-js/pull/2312 --- build.sbt | 19 +++++++++++++++++++ project/Build.scala | 45 ++++++++++++++++++++++----------------------- 2 files changed, 41 insertions(+), 23 deletions(-) create mode 100644 build.sbt diff --git a/build.sbt b/build.sbt new file mode 100644 index 000000000000..8f3754e57c33 --- /dev/null +++ b/build.sbt @@ -0,0 +1,19 @@ +val dotty = Build.dotty +val `dotty-bootstrapped` = Build.`dotty-bootstrapped` +val `dotty-interfaces` = Build.`dotty-interfaces` +val `dotty-doc` = Build.`dotty-doc` +val `dotty-bot` = Build.`dotty-bot` +val `dotty-compiler` = Build.`dotty-compiler` +val `dotty-compiler-bootstrapped` = Build.`dotty-compiler-bootstrapped` +val `dotty-bin-tests` = Build.`dotty-bin-tests` +val `dotty-library` = Build.`dotty-library` +val `dotty-library-bootstrapped` = Build.`dotty-library-bootstrapped` +val `dotty-sbt-bridge` = Build.`dotty-sbt-bridge` +val sjsSandbox = Build.sjsSandbox +val `dotty-bench` = Build.`dotty-bench` +val `scala-library` = Build.`scala-library` +val `scala-compiler` = Build.`scala-compiler` +val `scala-reflect` = Build.`scala-reflect` +val scalap = Build.scalap + +inThisBuild(Build.thisBuildSettings) diff --git a/project/Build.scala b/project/Build.scala index 10a39767830a..336cf9c81835 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -10,7 +10,7 @@ import org.scalajs.sbtplugin.ScalaJSPlugin import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport._ import sbt.Package.ManifestAttributes -object DottyBuild extends Build { +object Build { val scalacVersion = "2.11.5" // Do not rename, this is grepped in bin/common. @@ -59,26 +59,25 @@ object DottyBuild extends Build { // Shorthand for compiling a docs site lazy val dottydoc = inputKey[Unit]("run dottydoc") - override def settings: Seq[Setting[_]] = { - super.settings ++ Seq( - scalaVersion in Global := scalacVersion, - version in Global := dottyVersion, - organization in Global := dottyOrganization, - organizationName in Global := "LAMP/EPFL", - organizationHomepage in Global := Some(url("http://lamp.epfl.ch")), - homepage in Global := Some(url("https://github.com/lampepfl/dotty")), - - // scalac options - scalacOptions in Global ++= Seq( - "-feature", - "-deprecation", - "-encoding", "UTF8", - "-language:existentials,higherKinds,implicitConversions" - ), - - javacOptions in Global ++= Seq("-Xlint:unchecked", "-Xlint:deprecation") - ) - } + // Used in build.sbt + val thisBuildSettings = Seq( + scalaVersion in Global := scalacVersion, + version in Global := dottyVersion, + organization in Global := dottyOrganization, + organizationName in Global := "LAMP/EPFL", + organizationHomepage in Global := Some(url("http://lamp.epfl.ch")), + homepage in Global := Some(url("https://github.com/lampepfl/dotty")), + + // scalac options + scalacOptions in Global ++= Seq( + "-feature", + "-deprecation", + "-encoding", "UTF8", + "-language:existentials,higherKinds,implicitConversions" + ), + + javacOptions in Global ++= Seq("-Xlint:unchecked", "-Xlint:deprecation") + ) /** Enforce 2.11.5. Do not let it be upgraded by dependencies. */ private val overrideScalaVersionSetting = @@ -124,7 +123,7 @@ object DottyBuild extends Build { lazy val dotty = project.in(file(".")). // FIXME: we do not aggregate `bin` because its tests delete jars, thus breaking other tests aggregate(`dotty-interfaces`, `dotty-library`, `dotty-compiler`, `dotty-doc`, dottySbtBridgeRef, - `scala-library`, `scala-compiler`, `scala-reflect`, `scalap`). + `scala-library`, `scala-compiler`, `scala-reflect`, scalap). dependsOn(`dotty-compiler`). dependsOn(`dotty-library`). settings( @@ -728,7 +727,7 @@ object DottyInjectedPlugin extends AutoPlugin { libraryDependencies := Seq("org.scala-lang" % "scala-reflect" % scalaVersion.value) ). settings(publishing) - lazy val `scalap` = project. + lazy val scalap = project. settings( crossPaths := false, libraryDependencies := Seq("org.scala-lang" % "scalap" % scalaVersion.value)