Run Integration tests in velocity (#4204)

* Run Integration Tests in Velocity

- Build a base image (conditionally) if the build itself
  changed. This image already has SBT and all of the build dependencies
  downloaded. If possible, it will push/pull from dockerhub.
- Then run this with the jenkins workspace volume mounted and
  run all of the integration tests, etc.
  We may want to consider assembly as well.

- Disable docker integration tests in Velocity
- Change build status to velocity instead of Travis
- Disable the flakey tests
- Run a forked JVM for every integration test.
- Move more tools to use ephemeral ports.

Dockerfile.build

@@ -0,0 +1,43 @@
+#
+# Base docker image used to build inside of a CI environment.
+# This should be updated and pushed with bin/update-base-build.sh
+# when we change the SBT version, mesos lib, or dependencies
+#
+# While it is not absolutely necessary (with the exception of the
+# mesos lib), it will significantly speedup the build.
+#
+# We install docker 1.11.0 to /usr/bin - it likely should be overridden by
+# the user's docker so that the versions match
+#
+# CI Builds should base their build off of this image which is tagged
+# carefully
+
+FROM java:8-jdk
+
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E56151BF && \
+    echo "deb http://repos.mesosphere.com/debian jessie-testing main" | tee /etc/apt/sources.list.d/mesosphere.list && \
+    echo "deb http://repos.mesosphere.com/debian jessie main" | tee -a /etc/apt/sources.list.d/mesosphere.list && \
+    apt-get update && \
+    apt-get install --no-install-recommends -y --force-yes mesos=1.0.0-2.0.89.debian81 && \
+    curl -o /tmp/docker.tgz https://get.docker.com/builds/Linux/x86_64/docker-1.11.0.tgz && \
+    cd /tmp && \
+    tar zxf docker.tgz && \
+    mv docker/docker /usr/bin/docker && \
+    chmod +x /usr/bin/docker
+
+COPY . /marathon
+RUN rm -rf /marathon/project/project/target && \
+    rm -rf /marathon/project/target && \
+    eval $(sed s/sbt.version/SBT_VERSION/ < /marathon/project/build.properties) && \
+    mkdir -p /usr/local/bin && \
+    wget -P /usr/local/bin/ https://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/$SBT_VERSION/sbt-launch.jar && \
+    cp /marathon/project/sbt /usr/local/bin && \
+    chmod +x /usr/local/bin/sbt
+
+RUN cd /marathon && \
+    sbt -Dsbt.log.format=false update && \
+    rm -rf /marathon/*
+
+WORKDIR /marathon
+
+ENTRYPOINT ["/bin/bash"]

README.md

@@ -1,5 +1,5 @@
 [![Stories in Ready](https://badge.waffle.io/mesosphere/marathon.png?label=ready&title=Ready)](https://waffle.io/mesosphere/marathon)
-# [Marathon](https://mesosphere.github.io/marathon/) [![Build Status](https://travis-ci.org/mesosphere/marathon.png?branch=master)](https://travis-ci.org/mesosphere/marathon) [![Coverage Status](https://coveralls.io/repos/mesosphere/marathon/badge.svg?branch=master)](https://coveralls.io/r/mesosphere/marathon?branch=master)
+# [Marathon](https://mesosphere.github.io/marathon/) [![Build Status](https://jenkins.mesosphere.com/service/jenkins/job/public-marathon-snapshot-packages-master/badge/icon)](https://jenkins.mesosphere.com/service/jenkins/job/public-marathon-snapshot-packages-master) [![Coverage Status](https://coveralls.io/repos/mesosphere/marathon/badge.svg?branch=master)](https://coveralls.io/r/mesosphere/marathon?branch=master)
 
 Marathon is a production-proven [Apache Mesos][Mesos] framework for container orchestration. [DC/OS](https://dcos.io/get-started/#marathon) is the easiest way to start using Marathon.
 

bin/build-base-image.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+cd $(dirname $0)/..
+
+BASE_IMAGE_NAME=mesosphere/marathon-build-base
+BUILD_BASE_ID=$(md5sum Dockerfile.build *.sbt project/*.properties project/*.sbt project/*.scala | md5sum - | cut -f 1 -d ' ')
+TAGGED_IMAGE=$BASE_IMAGE_NAME:$BUILD_BASE_ID
+
+# Check if we need to build the base image
+if [[ "$(docker images -q $TAGGED_IMAGE 2>/dev/null)" = "" ]]; then
+  # try to pull, if that works, we're all set
+  if [[ "$DOCKER_HUB_USERNAME" != "" ]]; then
+    docker login -u "$DOCKER_HUB_USERNAME" -p "$DOCKER_HUB_PASSWORD" -e "$DOCKER_HUB_PASSWORD" >&2 2>/dev/null || true
+  fi
+  if ! docker pull $TAGGED_IMAGE 1>&2; then
+    docker build -t $TAGGED_IMAGE -f Dockerfile.build . >&2
+    if [[ "$DOCKER_HUB_USERNAME" != "" ]]; then
+      docker push $TAGGED_IMAGE >&2 || true
+    fi
+  fi
+  echo "Pulled $TAGGED_IMAGE" >&2
+fi
+
+echo $TAGGED_IMAGE

mesos-simulation/src/test/scala/mesosphere/mesos/scale/SingleAppScalingTest.scala

@@ -2,12 +2,12 @@ package mesosphere.mesos.scale
 
 import java.io.File
 
-import mesosphere.marathon.api.v2.json.{ AppUpdate }
-import mesosphere.marathon.integration.facades.{ ITDeploymentResult, MarathonFacade }
+import mesosphere.marathon.api.v2.json.AppUpdate
+import mesosphere.marathon.integration.facades.ITDeploymentResult
+import mesosphere.marathon.integration.facades.MarathonFacade._
 import mesosphere.marathon.integration.setup._
-import MarathonFacade._
 import mesosphere.marathon.state.{ AppDefinition, PathId }
-import org.scalatest.{ BeforeAndAfter, GivenWhenThen, Matchers }
+import org.scalatest.{ BeforeAndAfter, ConfigMap, GivenWhenThen, Matchers }
 import org.slf4j.LoggerFactory
 import play.api.libs.json._
 
@@ -30,6 +30,14 @@ class SingleAppScalingTest
   //clean up state before running the test case
   before(cleanUp())
 
+  override def afterAll(configMap: ConfigMap): Unit = {
+    super.afterAll(configMap)
+    println()
+    DisplayAppScalingResults.displayMetrics(SingleAppScalingTest.metricsFile)
+    println()
+    DisplayAppScalingResults.displayAppInfoScaling(SingleAppScalingTest.appInfosFile)
+  }
+
   override def startMarathon(port: Int, args: String*): Unit = {
     val cwd = new File(".")
 

project/build.scala

@@ -1,19 +1,20 @@
-import com.amazonaws.auth.{ EnvironmentVariableCredentialsProvider, InstanceProfileCredentialsProvider }
+import com.amazonaws.auth.{EnvironmentVariableCredentialsProvider, InstanceProfileCredentialsProvider}
 import com.typesafe.sbt.SbtScalariform
 import com.typesafe.sbt.SbtScalariform.ScalariformKeys
 import ohnosequences.sbt.SbtS3Resolver.autoImport._
-import org.scalastyle.sbt.ScalastylePlugin.{ buildSettings => styleSettings }
+import org.scalastyle.sbt.ScalastylePlugin.{buildSettings => styleSettings}
 import sbt.Keys._
+import sbt.Tests.SubProcess
 import sbt._
 import sbtassembly.AssemblyKeys._
 import sbtassembly.MergeStrategy
 import sbtbuildinfo.BuildInfoKeys._
-import sbtbuildinfo.{ BuildInfoKey, BuildInfoPlugin }
+import sbtbuildinfo.{BuildInfoKey, BuildInfoPlugin}
 import sbtrelease.ReleasePlugin.autoImport._
 import sbtrelease.ReleaseStateTransformations._
 import sbtrelease._
-import scala.util.Try
 
+import scala.util.Try
 import scalariform.formatter.preferences._
 
 object MarathonBuild extends Build {
@@ -99,10 +100,22 @@ object MarathonBuild extends Build {
       fork in Benchmark := true
     )
 
+  // Someday, these may be able to run in parallel but the integration tests in particular have places
+  // where they can kill each other _even_ when in different processes.
   lazy val integrationTestSettings = inConfig(IntegrationTest)(Defaults.testTasks) ++
     Seq(
       testOptions in IntegrationTest := Seq(formattingTestArg, Tests.Argument("-n", "mesosphere.marathon.IntegrationTest")),
-      parallelExecution in IntegrationTest := false
+      parallelExecution in IntegrationTest := false,
+      testForkedParallel in IntegrationTest := false,
+      testListeners in IntegrationTest := Seq(new JUnitXmlTestsListener((target.value / "integration").getAbsolutePath)),
+      testGrouping in IntegrationTest := (definedTests in IntegrationTest).value.map { test =>
+        Tests.Group(name = test.name, tests = Seq(test),
+          runPolicy = SubProcess(ForkOptions((javaHome in IntegrationTest).value,
+            (outputStrategy in IntegrationTest).value, Nil, Some(baseDirectory.value),
+            (javaOptions in IntegrationTest).value, (connectInput in IntegrationTest).value,
+            (envVars in IntegrationTest).value
+          )))
+      }
     )
 
   lazy val testSettings = Seq(

src/test/scala/mesosphere/marathon/api/v2/ModelValidationTest.scala

@@ -10,8 +10,8 @@ import mesosphere.marathon.state.Container._
 import mesosphere.marathon.state.PathId._
 import mesosphere.marathon.state._
 import org.apache.mesos.Protos.ContainerInfo.DockerInfo.Network
-import org.scalatest.{BeforeAndAfterAll, Matchers, OptionValues}
-import play.api.libs.json.{JsObject, Json}
+import org.scalatest.{ BeforeAndAfterAll, Matchers, OptionValues }
+import play.api.libs.json.{ JsObject, Json }
 
 import scala.collection.immutable.Seq
 

src/test/scala/mesosphere/marathon/integration/GracefulTaskKillIntegrationTest.scala

@@ -14,16 +14,17 @@ class GracefulTaskKillIntegrationTest
     with BeforeAndAfter
     with GivenWhenThen {
 
-  //clean up state before running the test case
-  before(cleanUp())
+  before {
+    cleanUp()
+  }
 
   // this command simulates a 'long terminating' application
   // note: Integration test does not interpret symbolic names (SIGTERM=15), therefore signal 15 is used.
   val taskKillGraceDuration = 4
   val taskKillGracePeriod = taskKillGraceDuration.seconds
   val appCommand: String = s"""trap \"sleep ${taskKillGraceDuration + 1}\" 15 && sleep 100000"""
 
-  test("create a 'long terminating' app with custom taskKillGracePeriod duration") {
+  ignore("create a 'long terminating' app with custom taskKillGracePeriod duration - https://github.com/mesosphere/marathon/issues/4214") {
     Given("a new 'long terminating' app with taskKillGracePeriod set to 10 seconds")
     val app = AppDefinition(
       testBasePath / "app",
@@ -55,7 +56,7 @@ class GracefulTaskKillIntegrationTest
     waitedForTaskKilledEvent.toMillis should be >= taskKillGracePeriod.toMillis
   }
 
-  test("create a 'short terminating' app with custom taskKillGracePeriod duration") {
+  ignore("create a 'short terminating' app with custom taskKillGracePeriod duration - https://github.com/mesosphere/marathon/issues/4214") {
     Given("a new 'short terminating' app with taskKillGracePeriod set to 10 seconds")
     val app = AppDefinition(
       testBasePath / "app",

src/test/scala/mesosphere/marathon/integration/LeaderIntegrationTest.scala

@@ -57,7 +57,7 @@ class LeaderIntegrationTest extends IntegrationFunSuite
     WaitTestSupport.waitUntil("the leader changes", 30.seconds) { marathon.leader().value != leader }
   }
 
-  test("it survives a small burn-in reelection test") {
+  ignore("it survives a small burn-in reelection test - https://github.com/mesosphere/marathon/issues/4215") {
     val random = new scala.util.Random
     for (_ <- 1 to 10) {
       Given("a leader")

src/test/scala/mesosphere/marathon/integration/TaskLostIntegrationTest.scala

@@ -8,15 +8,15 @@ import scala.concurrent.duration._
 
 class TaskLostIntegrationTest extends IntegrationFunSuite with WithMesosCluster with Matchers with GivenWhenThen with BeforeAndAfter {
 
-  before {
+  after {
     cleanUp()
     if (ProcessKeeper.hasProcess(slave2)) stopMesos(slave2)
     if (ProcessKeeper.hasProcess(master2)) stopMesos(master2)
     if (!ProcessKeeper.hasProcess(master1)) startMaster(master1)
     if (!ProcessKeeper.hasProcess(slave1)) startSlave(slave1)
   }
 
-  test("A task lost with mesos master failover will not kill the task") {
+  ignore("A task lost with mesos master failover will not kill the task - https://github.com/mesosphere/marathon/issues/4214") {
     Given("a new app")
     val app = appProxy(testBasePath / "app", "v1", instances = 1, withHealth = false)
     marathon.createAppV2(app)
@@ -75,7 +75,7 @@ class TaskLostIntegrationTest extends IntegrationFunSuite with WithMesosCluster
     waitForTasks(app.id, 1).head should be(task)
   }
 
-  test("A task lost with mesos master failover will expunge the task after gc timeout") {
+  test("A task lost with mesos master failover will expunge the task after gc timeout - https://github.com/mesosphere/marathon/issues/4212") {
     Given("a new app")
     val app = appProxy(testBasePath / "app", "v1", instances = 1, withHealth = false)
     marathon.createAppV2(app)

src/test/scala/mesosphere/marathon/integration/ZooKeeperTest.scala

@@ -14,21 +14,26 @@ class ZooKeeperTest extends IntegrationFunSuite with SingleMarathonIntegrationTe
 
   test("/marathon has OPEN_ACL_UNSAFE acls") {
     Given("a leader has been elected")
-    WaitTestSupport.waitUntil("a leader has been elected", 30.seconds) { marathon.leader().code == 200 }
-
     val watcher = new Watcher { override def process(event: WatchedEvent): Unit = {} }
     val zooKeeper = new ZooKeeper(config.zkHostAndPort, 30 * 1000, watcher)
+    try {
+      WaitTestSupport.waitUntil("a leader has been elected", 30.seconds) {
+        marathon.leader().code == 200
+      }
 
-    Then("the /leader node exists")
-    val stat = zooKeeper.exists(config.zkPath + "/leader", false)
-    Option(stat) should not be empty
+      Then("the /leader node exists")
+      val stat = zooKeeper.exists(config.zkPath + "/leader", false)
+      Option(stat) should not be empty
 
-    And("it has the default OPEN_ACL_UNSAFE permissions")
-    val acls = zooKeeper.getACL(config.zkPath + "/leader", stat)
-    val expectedAcl = new util.ArrayList[ACL]
-    expectedAcl.addAll(ZooDefs.Ids.OPEN_ACL_UNSAFE)
-    expectedAcl.addAll(ZooDefs.Ids.READ_ACL_UNSAFE)
-    acls.toArray.toSet should equal(expectedAcl.toArray.toSet)
+      And("it has the default OPEN_ACL_UNSAFE permissions")
+      val acls = zooKeeper.getACL(config.zkPath + "/leader", stat)
+      val expectedAcl = new util.ArrayList[ACL]
+      expectedAcl.addAll(ZooDefs.Ids.OPEN_ACL_UNSAFE)
+      expectedAcl.addAll(ZooDefs.Ids.READ_ACL_UNSAFE)
+      acls.toArray.toSet should equal(expectedAcl.toArray.toSet)
+    } finally {
+      zooKeeper.close()
+    }
   }
 }
 

src/test/scala/mesosphere/marathon/integration/setup/IntegrationTestConfig.scala

@@ -46,6 +46,8 @@ case class IntegrationTestConfig(
     //mesosLib: path to the native mesos lib. Defaults to /usr/local/lib/libmesos.dylib
     mesosLib: String,
 
+    mesosPort: Int,
+
     //the marathon host to use.
     marathonHost: String,
 
@@ -118,24 +120,25 @@ object IntegrationTestConfig {
     val zkHost = string("zkHost", unusedForExternalSetup("localhost"))
     val zkPort = int("zkPort", PortAllocator.ephemeralPort())
     val zkCredentials = config.getOptional[String]("zkCredentials")
-    val master = string("master", unusedForExternalSetup("127.0.0.1:5050"))
+    val mesosPort = int("mesosPort", PortAllocator.ephemeralPort())
+    val master = string("master", unusedForExternalSetup(s"127.0.0.1:$mesosPort"))
     val mesosLib = string("mesosLib", unusedForExternalSetup(defaultMesosLibConfig))
     val httpPort = int("httpPort", PortAllocator.ephemeralPort())
     val marathonHost = string("marathonHost", "localhost")
     val marathonBasePort = int("marathonPort", PortAllocator.ephemeralPort())
     val clusterSize = int("clusterSize", 3)
-    val marathonPorts = 0.to(clusterSize - 1).map(_ + marathonBasePort)
+    val marathonPorts = 0.until(clusterSize).map(_ => PortAllocator.ephemeralPort())
     val marathonGroup = PathId(string("marathonGroup", "/marathon_integration_test"))
 
     IntegrationTestConfig(
       cwd,
       useExternalSetup,
       zkHost, zkPort, zkCredentials,
-      master, mesosLib,
+      master, mesosLib, mesosPort,
       marathonHost, marathonBasePort, marathonGroup,
       httpPort,
       clusterSize,
-      marathonPorts)
+      marathonBasePort +: marathonPorts)
   }
 }
 

src/test/scala/mesosphere/marathon/integration/setup/ProcessKeeper.scala

@@ -64,7 +64,7 @@ object ProcessKeeper {
       sys.env, _.contains("binding to port"))
   }
 
-  def startMesosLocal(): Process = {
+  def startMesosLocal(port: Int): Process = {
     val mesosWorkDirForMesos: String = "/tmp/marathon-itest-mesos"
     val mesosWorkDirFile: File = new File(mesosWorkDirForMesos)
     FileUtils.deleteDirectory(mesosWorkDirFile)
@@ -73,7 +73,7 @@ object ProcessKeeper {
     val mesosEnv = setupMesosEnv(mesosWorkDirFile, mesosWorkDirForMesos)
     startProcess(
       "mesos",
-      Process(Seq("mesos-local", "--ip=127.0.0.1"), cwd = None, mesosEnv: _*),
+      Process(Seq("mesos-local", "--ip=127.0.0.1", s"--port=$port"), cwd = None, mesosEnv: _*),
       upWhen = _.toLowerCase.contains("registered with master"))
   }
 

src/test/scala/mesosphere/marathon/integration/setup/SingleMarathonIntegrationTest.scala

@@ -110,7 +110,7 @@ trait SingleMarathonIntegrationTest
     }
   }
 
-  protected def startMesos(): Unit = ProcessKeeper.startMesosLocal()
+  protected def startMesos(): Unit = ProcessKeeper.startMesosLocal(config.mesosPort)
 
   protected def createConfig(configMap: ConfigMap): IntegrationTestConfig = IntegrationTestConfig(configMap)
 

src/test/scala/mesosphere/marathon/upgrade/DeploymentPlanTest.scala

@@ -8,8 +8,8 @@ import mesosphere.marathon.state.AppDefinition.VersionInfo.FullVersionInfo
 import mesosphere.marathon.state.PathId._
 import mesosphere.marathon.state._
 import mesosphere.marathon.test.Mockito
-import org.apache.mesos.{Protos => mesos}
-import org.scalatest.{GivenWhenThen, Matchers}
+import org.apache.mesos.{ Protos => mesos }
+import org.scalatest.{ GivenWhenThen, Matchers }
 
 import scala.collection.immutable.Seq
 

tests/integration/unit-integration-tests.sh

@@ -1,5 +1,40 @@
 #!/bin/bash
-set -v
-mkdir -p $HOME/.sbt/launchers/0.13.8/
-test -r $HOME/.sbt/launchers/0.13.8/sbt-launch.jar || curl -L -o $HOME/.sbt/launchers/0.13.8/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.8/sbt-launch.jar
-java -Dfile.encoding=utf-8 -jar $HOME/.sbt/launchers/0.13.8/sbt-launch.jar -Dsbt.log.noformat=true "; clean; coverage; doc; assembly; coverageReport; coveralls"
+set -e
+
+if [[ -z $WORKSPACE ]]; then
+  WORKSPACE=$(realpath $(dirname $0)/../..)
+  TARGETS_DIR=$WORKSPACE/targets-docker
+else
+  TARGET=$WORKSPACE/target
+fi
+
+export DOCKER_HUB_USERNAME
+export DOCKER_HUB_PASSWORD
+export DOCKER_HUB_EMAIL
+TAGGED_IMAGE=$($WORKSPACE/bin/build-base-image.sh)
+
+if [[ -z ${RUN_DOCKER_INTEGRATION_TESTS+x} ]]; then
+  if [[ -n "$JENKINS_HOME" ]]; then
+    # velocity can't run these tests (docker in docker in docker)
+    RUN_DOCKER_INTEGRATION_TESTS="false"
+  else
+    if [[ -z ${DOCKER_MACHINE_NAME+x} ]]; then
+      RUN_DOCKER_INTEGRATION_TESTS="true"
+    else
+      RUN_DOCKER_INTEGRATION_TESTS="false"
+    fi
+  fi
+fi
+
+export MARATHON_MAX_TASKS_PER_OFFER="${MARATHON_MAX_TASKS_PER_OFFER-1}"
+
+
+DOCKER_OPTIONS="run --entrypoint=/bin/bash --rm --name marathon-itests-$BUILD_ID --net host --privileged -e TARGETS_DIR=$TARGETS_DIR -e RUN_DOCKER_INTEGRATION_TESTS=$RUN_DOCKER_INTEGRATION_TESTS -e MARATHON_MAX_TASKS_PER_OFFER=$MARATHON_MAX_TASKS_PER_OFFER -v $WORKSPACE:/marathon -v $TARGET:/marathon/target -v /var/run/docker.sock:/var/run/docker.sock -v /etc/hosts:/etc/hosts -i"
+
+DOCKER_CMD="/usr/local/bin/sbt -Dsbt.log.format=false coverage doc test integration:test coverageReport coveralls mesos-simulation/integration:test"
+
+DOCKER_ARGS="$DOCKER_OPTIONS $TAGGED_IMAGE $DOCKER_CMD"
+
+echo "Running docker $DOCKER_ARGS"
+
+docker $DOCKER_ARGS 2>&1