Merge branch 'reef'

.travis.yml

@@ -1,7 +1,5 @@
 language: java
 jdk:
-- openjdk7
-- oraclejdk7
 - oraclejdk8
 addons:
   postgresql: 9.3

build.gradle

@@ -1,42 +1,42 @@
-/* This project uses Java and Eclipse. */
+/* Gradle's build script */
+buildscript {
+  dependencies {
+    classpath 'org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.0.0'
+    classpath 'gradle.plugin.com.github.sherter.google-java-format:google-java-format-gradle-plugin:0.2'
+  }
+}
+
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
+import com.github.jengelman.gradle.plugins.shadow.transformers.ApacheLicenseResourceTransformer
 import com.github.sherter.googlejavaformatgradleplugin.GoogleJavaFormat
 import com.github.sherter.googlejavaformatgradleplugin.VerifyGoogleJavaFormat
 
+plugins {
+  id 'java' // or 'groovy' Must be explicitly applied
+  id 'com.github.johnrengelman.shadow' version '1.2.2'
+}
+/* This project uses Java and Eclipse. */
 apply plugin: "java"
 apply plugin: "eclipse"
 /* Coveralls with JaCoCo */
 apply plugin: 'jacoco'
 apply plugin: 'com.github.kt3k.coveralls'
-apply plugin: "com.github.sherter.google-java-format"
+/* Maven shade plugin equivalent */
+apply plugin: 'com.github.johnrengelman.shadow'
+apply plugin: 'com.github.sherter.google-java-format'
 
 /* Set up group and version info for myria */
 archivesBaseName = "myria"
 group = "edu.washington.escience.myria"
 version = "0.1"
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '2.3'
+    gradleVersion = '2.10'
 }
 
-/* We use Java 1.7 */
-sourceCompatibility = 1.7
-targetCompatibility = 1.7
-
-/* Gradle's build script */
-buildscript {
-  repositories {
-    mavenCentral()
-    maven {
-      url "https://plugins.gradle.org/m2/"
-    }
-  }
-
-  dependencies {
-    classpath 'org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.0.0'
-    classpath "gradle.plugin.com.github.sherter.google-java-format:google-java-format-gradle-plugin:0.2"
-  }
-}
+/* We use Java 1.8 */
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
 
 /* Tell gradle where the source code is located. */
 sourceSets {
@@ -53,28 +53,8 @@ sourceSets {
     java.srcDir 'test'
     output.classesDir = 'build/test'
   }
-  /* Code for systemtests */
-  systemtest {
-    java.srcDir 'systemtest'
-    output.classesDir = 'build/systemtest'
-  }
-  /* Code for speedtests */
-  speedtest {
-    java.srcDir 'speedtest'
-    output.classesDir = 'build/speedtest'
-  }
-  /* Code for testsuites */
-  testsuites {
-    java.srcDir 'testsuites'
-    output.classesDir = 'build/testsuites'
-  }
 }
 
-
-/* Compile, but don't run the speed tests when building the project. */
-check.dependsOn speedtestClasses
-check.dependsOn systemtestClasses
-check.dependsOn testsuitesClasses
 /* Build the Eclipse .classpath file first, because if Gradle's build fails
  * we'll want to use Eclipse to debug it. */
 test.dependsOn eclipseClasspath
@@ -83,16 +63,12 @@ tasks.eclipseClasspath.dependsOn cleanEclipseClasspath
 task format(type: GoogleJavaFormat) {
   source 'src'
   source 'test'
-  source 'systemtest'
-  source 'speedtest'
   include '**/*.java'
 }
 
 task verifyFormatting(type: VerifyGoogleJavaFormat) {
   source 'src'
   source 'test'
-  source 'systemtest'
-  source 'speedtest'
   include '**/*.java'
 }
 /* Test if all *.java have been formatted. Run ./gradlew format to format all */
@@ -106,12 +82,20 @@ test.dependsOn verifyFormatting
 /* Where we will fetch external JAR files from. */
 repositories {
   /* We need to override the official Apache repo for our custom Hadoop jars */
+  /* TODO: remove when we upgrade to Hadoop 2.8 (fixes HADOOP-12807) */
   maven { url "https://raw.github.com/uwescience/hadoop/mvn-repo" }
   mavenCentral() /* The standard Java Maven repository. */
   maven { url "http://clojars.org/repo" } /* Clojars, for MonetDB */
   flatDir {
     name 'localRepository'
   }
+  /* Apache snapshot repo, for REEF dev builds */
+  /* maven { url "http://repository.apache.org/snapshots/" } */
+  /* Our private maven repo for our REEF fork */
+  maven { url "https://raw.github.com/uwescience/reef/mvn-repo" }
+
+  /* For local REEF builds installed in the local Maven repo. */
+  /* mavenLocal() */
 }
 
 repositories.localRepository {
@@ -125,6 +109,9 @@ ext.jerseyVersion = "2.14"
 ext.swaggerVersion = "2.11"
 ext.swaggerMinorVersion = "1.3.11"
 ext.jmxetricsVersion = "1.0.8"
+ext.reefVersion = "0.15.0"
+/* for local or development REEF builds */
+/* ext.reefVersion = "0.16.0-SNAPSHOT" */
 ext.hadoopVersion = "2.7.2"
 
 dependencies {
@@ -148,7 +135,8 @@ dependencies {
   compile "com.almworks.sqlite4java:sqlite4java:1.0.392"
 
   compile "com.google.guava:guava:18.0"
-  compile "com.google.protobuf:protobuf-java:2.6.1"
+  /* this version is required for compatibility with REEF */
+  compile "com.google.protobuf:protobuf-java:2.5.0"
 
   compile "io.netty:netty:3.9.6.Final"
 
@@ -189,24 +177,16 @@ dependencies {
 
   compile "org.postgresql:postgresql:9.4-1202-jdbc41"
 
+  compile("org.apache.reef:reef-project:${reefVersion}") { changing = true }
+  compile("org.apache.reef:reef-common:${reefVersion}") { changing = true }
+  compile("org.apache.reef:reef-runtime-local:${reefVersion}") { changing = true }
+  compile("org.apache.reef:reef-runtime-yarn:${reefVersion}") { changing = true }
+  compile("org.apache.reef:wake:${reefVersion}") { changing = true }
+  compile("org.apache.reef:tang:${reefVersion}") { changing = true }
+
   testCompile "junit:junit:4.12"
   testCompile "org.assertj:assertj-core:1.7.1"
   testCompile "com.carrotsearch:junit-benchmarks:0.7.2"
-
-  systemtestCompile sourceSets.main.output
-  systemtestCompile configurations.testCompile
-  systemtestCompile sourceSets.test.output
-  systemtestRuntime configurations.testRuntime
-
-  speedtestCompile sourceSets.main.output
-  speedtestCompile configurations.systemtestCompile
-  speedtestCompile sourceSets.systemtest.output
-  speedtestRuntime configurations.systemtestRuntime
-
-  testsuitesCompile sourceSets.main.output
-  testsuitesCompile configurations.systemtestCompile
-  testsuitesCompile sourceSets.systemtest.output
-  testsuitesRuntime configurations.systemtestRuntime
 }
 
 /* Tell Java to emit all compiler warnings. */
@@ -224,20 +204,6 @@ tasks.withType(Test) {
   maxHeapSize = "2048m"
 }
 
-/* Tell gradle where additional tests & testsuites should look for classes. */
-task systemtest(type: Test) {
-  testClassesDir = sourceSets.systemtest.output.classesDir
-  classpath = sourceSets.systemtest.runtimeClasspath
-}
-task speedtest(type: Test) {
-  testClassesDir = sourceSets.speedtest.output.classesDir
-  classpath = sourceSets.speedtest.runtimeClasspath
-}
-task testsuites(type: Test) {
-  testClassesDir = sourceSets.testsuites.output.classesDir
-  classpath = sourceSets.testsuites.runtimeClasspath
-}
-
 /* The eclipseClasspath task will build a local Eclipse .classpath file. */
 eclipse.classpath {
   /* Ensure Eclipse compiled output appears in build directory. */
@@ -266,9 +232,6 @@ eclipse.classpath {
       cp.entries.findAll { it instanceof SourceFolder && it.path.startsWith("protobuf") }*.output = "build/main"
       cp.entries.findAll { it instanceof SourceFolder && it.path.startsWith("third-party-src") }*.output = "build/main"
       cp.entries.findAll { it instanceof SourceFolder && it.path.startsWith("test") }*.output = "build/test"
-      cp.entries.findAll { it instanceof SourceFolder && it.path.startsWith("systemtest") }*.output = "build/systemtest"
-      cp.entries.findAll { it instanceof SourceFolder && it.path.startsWith("speedtest") }*.output = "build/speedtest"
-      cp.entries.findAll { it instanceof SourceFolder && it.path.startsWith("testsuites") }*.output = "build/testsuites"
     }
 
   }
@@ -295,31 +258,33 @@ task symlinkMetricsJar << {
 tasks.symlinkMetricsJar.dependsOn copyToLib
 tasks.jar.dependsOn symlinkMetricsJar
 
-/* Make sure test runs before any other type of tests */
-systemtest.dependsOn test
-speedtest.dependsOn test
-
 /* allTests just runs all tests */
 task allTests
 allTests.dependsOn test
-allTests.dependsOn systemtest
-allTests.dependsOn speedtest
-
-/* Make check run system tests. */
-check.dependsOn systemtest
 
 /* Assemble the Myria jar into `build/libs` */
 jar {
   manifest {
     attributes("Implementation-Title" : "Myria",
-               "Implementation-Version": version,
-	       "Main-Class": "edu.washington.escience.myria.daemon.MasterDaemon")
+               "Implementation-Version": version)
   }
 }
 
+/* Build the uberjar containing all our dependencies */
+shadowJar {
+  /* Without this option: "org.apache.tools.zip.Zip64RequiredException: archive contains more than 65535 entries." */
+  zip64 true
+  /* Apache license files will cause weird issues on case-insensitive filesystems like Mac HFS */
+  exclude "LICENSE", "META-INF/*.RSA", "META-INF/*.SF","META-INF/*.DSA", "META-INF/license"
+  transform(ApacheLicenseResourceTransformer)
+  /* Some of our deps conflict with old versions in Hadoop distro */
+  relocate 'com.google.common', 'myriadeps.com.google.common'
+  relocate 'javax.ws.rs', 'myriadeps.javax.ws.rs'
+}
+
 /* Coveralls */
 jacocoTestReport {
-  executionData(test, systemtest)
+  executionData(test)
   reports {
     xml.enabled = true // coveralls plugin depends on xml format report
     html.enabled = true

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Thu Mar 05 11:59:27 PST 2015
+#Wed Jun 08 14:06:49 PDT 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.3-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip

gradlew

@@ -42,11 +42,6 @@ case "`uname`" in
     ;;
 esac
 
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
 # Attempt to set APP_HOME
 # Resolve links: $0 may be a link
 PRG="$0"
@@ -61,9 +56,9 @@ while [ -h "$PRG" ] ; do
     fi
 done
 SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
+cd "`dirname \"$PRG\"`/" >/dev/null
 APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
+cd "$SAVED" >/dev/null
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
@@ -114,6 +109,7 @@ fi
 if $cygwin ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`