From bf161da15e56b53933aede4653eab4258efdb864 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Wed, 16 Mar 2016 15:40:23 -0700 Subject: [PATCH 1/4] Don't automatically exclude companion objects of private[spark] classes. --- project/MimaBuild.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala index 4adf64a5a0d86..acf7b8961eb74 100644 --- a/project/MimaBuild.scala +++ b/project/MimaBuild.scala @@ -42,14 +42,11 @@ object MimaBuild { ProblemFilters.exclude[IncompatibleFieldTypeProblem](fullName) ) - // Exclude a single class and its corresponding object + // Exclude a single class def excludeClass(className: String) = Seq( excludePackage(className), ProblemFilters.exclude[MissingClassProblem](className), - ProblemFilters.exclude[MissingTypesProblem](className), - excludePackage(className + "$"), - ProblemFilters.exclude[MissingClassProblem](className + "$"), - ProblemFilters.exclude[MissingTypesProblem](className + "$") + ProblemFilters.exclude[MissingTypesProblem](className) ) // Exclude a Spark class, that is in the package org.apache.spark From 77613e633bfa9fcf9f6117a5b6307fe3611ed00c Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Wed, 16 Mar 2016 16:40:37 -0700 Subject: [PATCH 2/4] Use previous artifact JARs to generate excludes --- dev/mima | 18 +++++++----------- project/SparkBuild.scala | 23 ++++++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/dev/mima b/dev/mima index c8e2df6cfcd4f..ea746e6f01b49 100755 --- a/dev/mima +++ b/dev/mima @@ -24,21 +24,17 @@ set -e FWDIR="$(cd "`dirname "$0"`"/..; pwd)" cd "$FWDIR" +SPARK_PROFILES="-Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive" TOOLS_CLASSPATH="$(build/sbt "export tools/fullClasspath" | tail -n1)" +OLD_DEPS_CLASSPATH="$(build/sbt $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)" rm -f .generated-mima* -generate_mima_ignore() { - java \ - -XX:MaxPermSize=1g \ - -Xmx2g \ - -cp "$TOOLS_CLASSPATH:$1" \ - org.apache.spark.tools.GenerateMIMAIgnore -} - -SPARK_PROFILES="-Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive" -generate_mima_ignore "$(build/sbt $SPARK_PROFILES "export assembly/fullClasspath" | tail -n1)" -generate_mima_ignore "$(build/sbt $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)" +java \ + -XX:MaxPermSize=1g \ + -Xmx2g \ + -cp "$TOOLS_CLASSPATH:$OLD_DEPS_CLASSPATH" \ + org.apache.spark.tools.GenerateMIMAIgnore echo -e "q\n" | build/sbt mimaReportBinaryIssues | grep -v -e "info.*Resolving" ret_val=$? diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index f76cda08ec36a..9782b3922402b 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -26,6 +26,7 @@ import sbt.Classpaths.publishTask import sbt.Keys._ import sbtunidoc.Plugin.UnidocKeys.unidocGenjavadocVersion import com.typesafe.sbt.pom.{PomBuild, SbtPomKeys} +import com.typesafe.tools.mima.plugin.MimaKeys import spray.revolver.RevolverPlugin._ @@ -247,12 +248,14 @@ object SparkBuild extends PomBuild { /* Enable tests settings for all projects except examples, assembly and tools */ (allProjects ++ optionallyEnabledProjects).foreach(enable(TestSettings.settings)) - allProjects.filterNot { x => + val mimaProjects = allProjects.filterNot { x => Seq( spark, hive, hiveThriftServer, catalyst, repl, networkCommon, networkShuffle, networkYarn, unsafe, testTags, sketch ).contains(x) - }.foreach { x => + } + + mimaProjects.foreach { x => enable(MimaBuild.mimaSettings(sparkHome, x))(x) } @@ -371,22 +374,24 @@ object ExcludedDependencies { } /** - * Following project only exists to pull previous artifacts of Spark for generating + * project only exists to pull previous artifacts of Spark for generating * Mima ignores. For more information see: SPARK 2071 */ object OldDeps { lazy val project = Project("oldDeps", file("dev"), settings = oldDepsSettings) + lazy val allPreviousArtifactKeys = Def.settingDyn[Seq[Option[ModuleID]]] { + SparkBuild.mimaProjects + .map { project => MimaKeys.previousArtifact in project } + .map(k => Def.setting(k.value)) + .join + } + def oldDepsSettings() = Defaults.coreDefaultSettings ++ Seq( name := "old-deps", scalaVersion := "2.10.5", - libraryDependencies := Seq( - "spark-streaming", - "spark-mllib", - "spark-graphx", - "spark-core" - ).map(id => "org.apache.spark" % (id + "_2.11") % "1.2.0") + libraryDependencies := allPreviousArtifactKeys.value.flatten ) } From 4605c79b7ea65735f0c1356e95f5f348b70d40d7 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Wed, 16 Mar 2016 16:49:41 -0700 Subject: [PATCH 3/4] Add MiMa excludes so checks pass for now. --- project/MimaExcludes.scala | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 59c7e7db2e1e2..18839a5423557 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -224,6 +224,7 @@ object MimaExcludes { ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.streaming.receiver.ActorReceiver$Supervisor") ) ++ Seq( // SPARK-12348 Remove deprecated Streaming APIs. + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.streaming.dstream.DStream.foreach"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.streaming.StreamingContext.toPairDStreamFunctions"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.streaming.StreamingContext.toPairDStreamFunctions$default$4"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.streaming.StreamingContext.awaitTermination"), @@ -535,6 +536,24 @@ object MimaExcludes { // SPARK-13927: add row/column iterator to local matrices ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.mllib.linalg.Matrix.rowIter"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.mllib.linalg.Matrix.colIter") + ) ++ Seq( + // SPARK-13948: MiMa Check should catch if the visibility change to `private` + // TODO(josh): Some of these may be legitimate incompatibilities; we should follow up before the 2.0.0 release + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.Dataset.toDS"), + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.sources.OutputWriterFactory.newInstance"), + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.util.RpcUtils.askTimeout"), + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.util.RpcUtils.lookupTimeout"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.UnaryTransformer.transform"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.classification.DecisionTreeClassifier.train"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.classification.LogisticRegression.train"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ml.regression.DecisionTreeRegressor.train"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.Dataset.groupBy"), + ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.Dataset.groupBy"), + ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.Dataset.select"), + ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.sql.Dataset.toDF"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.Logging.initializeLogIfNecessary"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerEvent.logEvent"), + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.sources.OutputWriterFactory.newInstance") ) case v if v.startsWith("1.6") => Seq( From fbb93ca0bdd979851e88270cb1b055c7276d0d42 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Wed, 16 Mar 2016 17:16:04 -0700 Subject: [PATCH 4/4] comment update --- project/SparkBuild.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 9782b3922402b..dbe98d1e14fda 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -374,8 +374,7 @@ object ExcludedDependencies { } /** - * project only exists to pull previous artifacts of Spark for generating - * Mima ignores. For more information see: SPARK 2071 + * Project to pull previous artifacts of Spark for generating Mima excludes. */ object OldDeps {