Permalink
Browse files

update build to prep for 2.12 (#700)

* update build to prep for 2.12

* record current binary incompatibilities

* fix logging

* change logger

* bump business in sbt
  • Loading branch information...
1 parent c302fc1 commit 9a80b22e1d970ab1f72448cea489834308b54e02 @sritchie sritchie committed on GitHub Dec 12, 2016
Showing with 139 additions and 249 deletions.
  1. +12 −0 .jvmopts
  2. +32 −103 .travis.yml
  3. +6 −1 README.md
  4. +72 −18 build.sbt
  5. +0 −44 project/DocGen.scala
  6. +0 −54 project/Unidoc.scala
  7. +1 −1 project/build.properties
  8. +9 −14 project/plugins.sbt
  9. +5 −1 project/travis-log4j.properties
  10. +2 −2 sbt
  11. +0 −11 scripts/run_tests.sh
View
@@ -0,0 +1,12 @@
+-Dfile.encoding=UTF8
+-Xms1G
+-Xmx3G
+-Xss2m
+-XX:MaxPermSize=512M
+-XX:ReservedCodeCacheSize=250M
+-XX:+TieredCompilation
+-XX:-UseGCOverheadLimit
+# effectively adds GC to Perm space
+-XX:+CMSClassUnloadingEnabled
+# must be enabled for CMSClassUnloadingEnabled to work
+-XX:+UseConcMarkSweepGC
View
@@ -1,109 +1,38 @@
-#test execution script.
-
language: scala
sudo: false
before_install: umask 0022
-scala:
- - 2.10.5
- - 2.11.7
-script:
-- "echo no op"
+env:
+ global:
+ - LOG4J=file://$TRAVIS_BUILD_DIR/project/travis-log4j.properties
matrix:
include:
-#BASE TESTS
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-batch"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-batch-hadoop"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-builder"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-chill"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-core"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-core-test"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-example"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-online"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-scalding"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-scalding-test"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-storm"
- script: "scripts/run_tests.sh"
-
- - scala: 2.10.5
- env: BUILD="base" TEST_TARGET="summingbird-storm-test"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-batch"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-batch-hadoop"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-builder"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-chill"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-core"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-core-test"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-example"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-online"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-scalding"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-scalding-test"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-storm"
- script: "scripts/run_tests.sh"
-
- - scala: 2.11.7
- env: BUILD="base" TEST_TARGET="summingbird-storm-test"
- script: "scripts/run_tests.sh"
+ - scala: 2.10.6
+ script: ./sbt -Dlog4j.configuration=$LOG4J ++$TRAVIS_SCALA_VERSION test mimaReportBinaryIssues
+
+ - scala: 2.11.8
+ script: ./sbt -Dlog4j.configuration=$LOG4J ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport mimaReportBinaryIssues
+ after_success:
+ - bash <(curl -s https://codecov.io/bash)
+
+cache:
+ directories:
+ - $HOME/.sbt/0.13/dependency
+ - $HOME/.sbt/boot/scala*
+ - $HOME/.sbt/launchers
+ - $HOME/.ivy2/cache
+ - $HOME/.nvm
+
+before_cache:
+ - du -h -d 1 $HOME/.ivy2/cache
+ - du -h -d 2 $HOME/.sbt/
+ - find $HOME/.sbt -name "*.lock" -type f -delete
+ - find $HOME/.ivy2/cache -name "ivydata-*.properties" -type f -delete
+
+notifications:
+ webhooks:
+ urls:
+ - https://webhooks.gitter.im/e/535b6396650b3a9c54a7
+ on_success: change
+ on_failure: always
+ on_start: never
View
@@ -1,4 +1,9 @@
-## Summingbird [![Build Status](https://secure.travis-ci.org/twitter/summingbird.png)](http://travis-ci.org/twitter/summingbird)
+## Summingbird
+
+[![Build Status](https://secure.travis-ci.org/twitter/summingbird.png)](http://travis-ci.org/twitter/summingbird)
+[![Codecov branch](https://img.shields.io/codecov/c/github/twitter/summingbird/develop.svg?maxAge=3600)](https://codecov.io/github/twitter/summingbird)
+[![Latest version](https://index.scala-lang.org/twitter/summingbird/summingbird-core/latest.svg?color=orange)](https://index.scala-lang.org/twitter/summingbird/summingbird-core)
+[![Chat](https://badges.gitter.im/twitter/summingbird.svg)](https://gitter.im/twitter/summingbird?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
Summingbird is a library that lets you write MapReduce programs that look like native Scala or Java collection transformations and execute them on a number of well-known distributed MapReduce platforms, including [Storm](https://github.com/nathanmarz/storm) and [Scalding](https://github.com/twitter/scalding).
View
@@ -1,11 +1,9 @@
import AssemblyKeys._
import ReleaseTransformations._
import com.typesafe.sbt.SbtScalariform._
-import com.typesafe.tools.mima.plugin.MimaKeys.previousArtifact
import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings
import sbtassembly.Plugin._
import scalariform.formatter.preferences._
-import summingbird._
def scalaBinaryVersion(scalaVersion: String) = scalaVersion match {
case version if version startsWith "2.10" => "2.10"
@@ -35,7 +33,7 @@ val stormDep = "storm" % "storm" % "0.9.0-wip15" //This project also compiles wi
val tormentaVersion = "0.11.1"
val utilVersion = "6.34.0"
-val extraSettings = Project.defaultSettings ++ mimaDefaultSettings ++ scalariformSettings
+val extraSettings = mimaDefaultSettings ++ scalariformSettings
val sharedSettings = extraSettings ++ Seq(
organization := "com.twitter",
@@ -100,15 +98,12 @@ val sharedSettings = extraSettings ++ Seq(
ReleaseStep(action = Command.process("sonatypeReleaseAll", _)),
pushChanges),
- publishTo <<= version { v =>
- Some(
- if (v.trim.toUpperCase.endsWith("SNAPSHOT"))
+ publishTo := Some(
+ if (version.value.trim.toUpperCase.endsWith("SNAPSHOT"))
Opts.resolver.sonatypeSnapshots
else
Opts.resolver.sonatypeStaging
- //"twttr" at "http://artifactory.local.twitter.com/libs-releases-local"
- )
- },
+ ),
pomExtra := (
<url>https://github.com/twitter/summingbird</url>
@@ -150,15 +145,19 @@ lazy val formattingPreferences = {
setPreference(PreserveSpaceBeforeArguments, true)
}
+lazy val noPublishSettings = Seq(
+ publish := (),
+ publishLocal := (),
+ test := (),
+ publishArtifact := false
+ )
+
lazy val summingbird = Project(
id = "summingbird",
base = file("."),
- settings = sharedSettings ++ DocGen.publishSettings
-).settings(
- test := { },
- publish := { }, // skip publishing for this root project.
- publishLocal := { }
-).aggregate(
+ settings = sharedSettings)
+ .settings(noPublishSettings)
+ .aggregate(
summingbirdCore,
summingbirdBatch,
summingbirdBatchHadoop,
@@ -185,12 +184,68 @@ def youngestForwardCompatible(subProj: String) =
.filterNot(unreleasedModules.contains(_))
.map { s => "com.twitter" % ("summingbird-" + s + "_2.10") % "0.9.0" }
+/**
+ * Empty this each time we publish a new version (and bump the minor number)
+ */
+val ignoredABIProblems = {
+ import com.typesafe.tools.mima.core._
+ import com.typesafe.tools.mima.core.ProblemFilters._
+ Seq(
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.memory.Memory.toStream"),
+ exclude[ReversedMissingMethodProblem]("com.twitter.summingbird.planner.DagOptimizer.FlatMapValuesFusion"),
+ exclude[ReversedMissingMethodProblem]("com.twitter.summingbird.planner.DagOptimizer.FlatMapKeyFusion"),
+ exclude[IncompatibleResultTypeProblem]("com.twitter.summingbird.batch.store.HDFSMetadata.versionedStore"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.planner.OnlinePlan.this"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.planner.StripNamedNode.castTail"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.planner.StripNamedNode.functionize"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.planner.StripNamedNode.castToPair"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.planner.StripNamedNode.processLevel"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.planner.StripNamedNode.toFunctional"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.planner.StripNamedNode.mutateGraph"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.planner.StripNamedNode.stripNamedNodes"),
+ exclude[ReversedMissingMethodProblem]("com.twitter.summingbird.online.OnlineDefaultConstants.com$twitter$summingbird$online$OnlineDefaultConstants$_setter_$DEFAULT_FM_MERGEABLE_WITH_SOURCE_="),
+ exclude[ReversedMissingMethodProblem]("com.twitter.summingbird.online.OnlineDefaultConstants.DEFAULT_FM_MERGEABLE_WITH_SOURCE"),
+ exclude[MissingClassProblem]("com.twitter.summingbird.online.MergeableStoreFactoryAlgebra"),
+ exclude[MissingClassProblem]("com.twitter.summingbird.online.MergeableStoreFactoryAlgebra$"),
+ exclude[IncompatibleResultTypeProblem]("com.twitter.summingbird.online.MergeableStoreFactory.mergeableStore"),
+ exclude[ReversedMissingMethodProblem]("com.twitter.summingbird.online.MergeableStoreFactory.mergeableStore"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.OperationContainer.init"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.OperationContainer.decoder"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.OperationContainer.encoder"),
+ exclude[ReversedMissingMethodProblem]("com.twitter.summingbird.online.executor.OperationContainer.init"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.FinalFlatMap.decoder"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.FinalFlatMap.encoder"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.FinalFlatMap.this"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.IntermediateFlatMap.decoder"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.IntermediateFlatMap.encoder"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.IntermediateFlatMap.this"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.AsyncBase.init"),
+ exclude[IncompatibleResultTypeProblem]("com.twitter.summingbird.online.executor.AsyncBase.execute"),
+ exclude[IncompatibleResultTypeProblem]("com.twitter.summingbird.online.executor.AsyncBase.executeTick"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.AsyncBase.logger"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.Summer.init"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.Summer.this"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.Summer.decoder"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.online.executor.Summer.encoder"),
+ exclude[MissingClassProblem]("com.twitter.summingbird.scalding.LookupJoin"),
+ exclude[MissingClassProblem]("com.twitter.summingbird.scalding.LookupJoin$"),
+ exclude[IncompatibleResultTypeProblem]("com.twitter.summingbird.storm.BaseBolt.copy$default$8"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.storm.BaseBolt.copy"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.storm.BaseBolt.this"),
+ exclude[IncompatibleMethTypeProblem]("com.twitter.summingbird.storm.Storm.get"),
+ exclude[IncompatibleMethTypeProblem]("com.twitter.summingbird.storm.Storm.getOrElse"),
+ exclude[DirectMissingMethodProblem]("com.twitter.summingbird.storm.BaseBolt.apply"),
+ exclude[IncompatibleResultTypeProblem]("com.twitter.summingbird.example.Memcache.client")
+ )
+}
+
def module(name: String) = {
val id = "summingbird-%s".format(name)
Project(id = id, base = file(id), settings = sharedSettings ++ Seq(
Keys.name := id,
- previousArtifact := youngestForwardCompatible(name))
- )
+ mimaPreviousArtifacts := youngestForwardCompatible(name).toSet,
+ mimaBinaryIssueFilters ++= ignoredABIProblems
+ ))
}
lazy val summingbirdBatch = module("batch").settings(
@@ -369,4 +424,3 @@ lazy val summingbirdCoreTest = module("core-test").settings(
).dependsOn(
summingbirdCore % "test->test;compile->compile"
)
-
View
@@ -1,44 +0,0 @@
-package summingbird
-
-import sbt._
-import Keys._
-
-import com.typesafe.sbt.git.GitRunner
-import com.typesafe.sbt.SbtGit.GitKeys
-import com.typesafe.sbt.SbtSite.{ site, SiteKeys }
-import com.typesafe.sbt.SbtGhPages.{ ghpages, GhPagesKeys => ghkeys }
-import com.typesafe.sbt.SbtGit.GitKeys.gitRemoteRepo
-
-object DocGen {
- val docDirectory = "target/site"
- val aggregateName = "summingbird"
-
- def syncLocal = (ghkeys.updatedRepository, GitKeys.gitRunner, streams) map { (repo, git, s) =>
- cleanSite(repo, git, s) // First, remove 'stale' files.
- val rootPath = file(docDirectory) // Now copy files.
- IO.copyDirectory(rootPath, repo)
- IO.touch(repo / ".nojekyll")
- repo
- }
-
- private def cleanSite(dir: File, git: GitRunner, s: TaskStreams): Unit = {
- val toClean = IO.listFiles(dir).filterNot(_.getName == ".git").map(_.getAbsolutePath).toList
- if(!toClean.isEmpty)
- git(("rm" :: "-r" :: "-f" :: "--ignore-unmatch" :: toClean) :_*)(dir, s.log)
- ()
- }
-
- lazy val unidocSettings: Seq[sbt.Setting[_]] =
- site.includeScaladoc(docDirectory) ++ Seq(
- scalacOptions in doc <++= (version, baseDirectory in LocalProject(aggregateName)).map { (v, rootBase) =>
- val tagOrBranch = if (v.endsWith("-SNAPSHOT")) "develop" else v
- val docSourceUrl = "https://github.com/twitter/" + aggregateName + "/tree/" + tagOrBranch + "€{FILE_PATH}.scala"
- Seq("-sourcepath", rootBase.getAbsolutePath, "-doc-source-url", docSourceUrl)
- },
- Unidoc.unidocDirectory := file(docDirectory),
- gitRemoteRepo := "git@github.com:twitter/" + aggregateName + ".git",
- ghkeys.synchLocal <<= syncLocal
- )
-
- lazy val publishSettings = site.settings ++ Unidoc.settings ++ ghpages.settings ++ unidocSettings
-}
Oops, something went wrong.

0 comments on commit 9a80b22

Please sign in to comment.