[SW-1422] Switch to one version

h2oai · Jul 15, 2019 · 876a954 · 876a954
1 parent 367bf76
commit 876a954
Show file tree

Hide file tree

Showing 18 changed files with 120 additions and 135 deletions.
diff --git a/build.gradle b/build.gradle
@@ -107,17 +107,15 @@ def loadVersion(versionSpecificProps){
 //
 // For all projects (this and all subprojects) specify common properties and tasks
 //
+def loadedVersion = loadVersion(versionSpecificProps).toString()
+
 configure(allprojects) { project ->
     apply plugin: 'idea'
     apply plugin: 'eclipse'
     apply from: "$rootDir/gradle/artifacts.gradle"
     apply plugin: 'nebula.lint'
     gradleLint.rules = ['all-dependency']
     gradleLint.alwaysRun = false // Run linter manually
-
-    // All project inherits the same versioning number
-    version = loadVersion(versionSpecificProps).toString().replace("-SNAPSHOT", "")
-
     // Version of main components
     ext {
         scalaBaseVersion = project.findProperty('scalaBaseVersion') ?: defaultScalaBaseVersion
@@ -133,10 +131,12 @@ configure(allprojects) { project ->
         supportedEmrVersion = loadVersionSpecificProp("supportedEmrVersion", versionSpecificProps)
         extraJVMArgs = loadVersionSpecificProp("extraJVMArgs", versionSpecificProps)
         unsupportedMinorSparkVersions = loadVersionSpecificProp("unsupportedMinorSparkVersions", versionSpecificProps)
-        // Nightly build versions have format x.y.z-SNAPSHOT-w where w is the SNAPSHOT build number
-        // After removing -SNAPSHOT we check for the -
-        isNightlyBuild = version.contains("-")
+        // Nightly build versions have format x.y.z-w-sparkMajor where w is the SNAPSHOT build number
+        isNightlyBuild = loadedVersion.substring(0, loadedVersion.lastIndexOf("-")).contains("-")
     }
+
+    // All project inherits the same versioning number
+    version = loadedVersion.replace("-SNAPSHOT", "")
 }
 
 // 

diff --git a/dist/build.gradle b/dist/build.gradle
@@ -206,3 +206,9 @@ task dist(dependsOn: zipDistribution) {
         substituteIndexHtml(buildTimeLocal)
     }
 }
+
+task s3Bucket {
+    doLast {
+       println(getS3BucketPrefix())
+    }
+}
diff --git a/dist/src/index.html b/dist/src/index.html
@@ -423,7 +423,7 @@ <h2>Integration info</h2>
                   <ul>
                     <li>H2O version: <a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/index.html">SUBST_H2O_VERSION.SUBST_H2O_BUILD SUBST_H2O_NAME</a> (<a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/docs-website/h2o-docs/index.html">documentation</a>)</li>
                     <li>Spark version: <a href="https://spark.apache.org/downloads.html">SUBST_SPARK_MAJOR_VERSION_WITH_EXCLUDES</a> (<a href="https://spark.apache.org/docs/SUBST_SPARK_VERSION/">documentation</a>)</li>
-                    <li>Sparkling Water: <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/index.html">Documentation</a> <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/CHANGELOG.html">Changelog</a></li>
+                    <li>Sparkling Water: <a href="doc/index.html">Documentation</a> <a href="doc/CHANGELOG.html">Changelog</a></li>
                     <li>Release date: SUBST_BUILD_TIME_LOCAL</li>
                   </ul>
                 </div>
@@ -465,7 +465,7 @@ <h2>Integration info</h2>
                   <ul>
                     <li>H2O version: <a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/index.html">SUBST_H2O_VERSION.SUBST_H2O_BUILD SUBST_H2O_NAME</a> (<a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/docs-website/h2o-docs/index.html">documentation</a>)</li>
                     <li>Spark version: <a href="https://spark.apache.org/downloads.html">SUBST_SPARK_MAJOR_VERSION_WITH_EXCLUDES</a> (<a href="https://spark.apache.org/docs/SUBST_SPARK_VERSION/">documentation</a>)</li>
-                    <li>Sparkling Water: <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/index.html">Documentation</a> <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/CHANGELOG.html">Changelog</a></li>
+                    <li>Sparkling Water: <a href="doc/index.html">Documentation</a> <a href="doc/CHANGELOG.html">Changelog</a></li>
                     <li>Release date: SUBST_BUILD_TIME_LOCAL</li>
                   </ul>
                 </div>
@@ -509,7 +509,7 @@ <h2>Integration info</h2>
                   <ul>
                     <li>H2O version: <a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/index.html">SUBST_H2O_VERSION.SUBST_H2O_BUILD SUBST_H2O_NAME</a> (<a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/docs-website/h2o-docs/index.html">documentation</a>)</li>
                     <li>Spark version: <a href="https://spark.apache.org/downloads.html">SUBST_SPARK_MAJOR_VERSION_WITH_EXCLUDES</a> (<a href="https://spark.apache.org/docs/SUBST_SPARK_VERSION/">documentation</a>)</li>
-                    <li>Sparkling Water: <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/index.html">Documentation</a> <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/CHANGELOG.html">Changelog</a></li>
+                    <li>Sparkling Water: <a href="doc/index.html">Documentation</a> <a href="doc/CHANGELOG.html">Changelog</a></li>
                     <li>Release date: SUBST_BUILD_TIME_LOCAL</li>
                   </ul>
                 </div>
@@ -548,7 +548,7 @@ <h2>Integration info</h2>
                 <ul>
                   <li>H2O version: <a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/index.html">SUBST_H2O_VERSION.SUBST_H2O_BUILD SUBST_H2O_NAME</a> (<a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/docs-website/h2o-docs/index.html">documentation</a>)</li>
                   <li>Spark version: <a href="https://spark.apache.org/downloads.html">SUBST_SPARK_MAJOR_VERSION_WITH_EXCLUDES</a> (<a href="https://spark.apache.org/docs/SUBST_SPARK_VERSION/">documentation</a>)</li>
-                  <li>Sparkling Water: <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/index.html">Documentation</a> <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/CHANGELOG.html">Changelog</a></li>
+                  <li>Sparkling Water: <a href="doc/index.html">Documentation</a> <a href="doc/CHANGELOG.html">Changelog</a></li>
                   <li>Release date: SUBST_BUILD_TIME_LOCAL</li>
                 </ul>
               </div>
@@ -586,7 +586,7 @@ <h2>Kluster</h2>
 
             <p>List of supported Hadoop distributions: SUBST_H2O_DRIVERS_LIST</p>
 
-            <p>For more info, please follow <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/deployment/backends.html">Kluster documentation</a>.</p>
+            <p>For more info, please follow <a href="doc/deployment/backends.html">Kluster documentation</a>.</p>
 
           </div>
           <div id="maven" style="display:none">
@@ -596,7 +596,7 @@ <h2>Integration info</h2>
                   <ul>
                     <li>H2O version: <a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/index.html">SUBST_H2O_VERSION.SUBST_H2O_BUILD SUBST_H2O_NAME</a> (<a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/docs-website/h2o-docs/index.html">documentation</a>)</li>
                     <li>Spark version: <a href="https://spark.apache.org/downloads.html">SUBST_SPARK_MAJOR_VERSION_WITH_EXCLUDES</a> (<a href="https://spark.apache.org/docs/SUBST_SPARK_VERSION/">documentation</a>)</li>
-                    <li>Sparkling Water: <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/index.html">Documentation</a> <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/CHANGELOG.html">Changelog</a></li>
+                    <li>Sparkling Water: <a href="doc/index.html">Documentation</a> <a href="doc/CHANGELOG.html">Changelog</a></li>
                     <li>Release date: SUBST_BUILD_TIME_LOCAL</li>
                   </ul>
                 </div>
@@ -621,7 +621,7 @@ <h2>Integration info</h2>
                   <ul>
                     <li>H2O version: <a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/index.html">SUBST_H2O_VERSION.SUBST_H2O_BUILD SUBST_H2O_NAME</a> (<a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/docs-website/h2o-docs/index.html">documentation</a>)</li>
                     <li>Spark version: <a href="https://spark.apache.org/downloads.html">SUBST_SPARK_MAJOR_VERSION_WITH_EXCLUDES</a> (<a href="https://spark.apache.org/docs/SUBST_SPARK_VERSION/">documentation</a>)</li>
-                    <li>Sparkling Water: <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/index.html">Documentation</a> <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/CHANGELOG.html">Changelog</a></li>
+                    <li>Sparkling Water: <a href="doc/index.html">Documentation</a> <a href="doc/CHANGELOG.html">Changelog</a></li>
                     <li>Release date: SUBST_BUILD_TIME_LOCAL</li>
                   </ul>
                 </div>
@@ -682,7 +682,7 @@ <h2>RSparkling</h2>
             </p>
 
 
-            <p>For more and detailed information, please follow the installation and usage instructions on <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/rsparkling.html">RSparkling page</a></p>
+            <p>For more and detailed information, please follow the installation and usage instructions on <a href="doc/rsparkling.html">RSparkling page</a></p>
 
 
             <h2>H2O R Client</h2>
@@ -697,7 +697,7 @@ <h2>Integration info</h2>
                 <ul>
                   <li>H2O version: <a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/index.html">SUBST_H2O_VERSION.SUBST_H2O_BUILD SUBST_H2O_NAME</a> (<a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/docs-website/h2o-docs/index.html">documentation</a>)</li>
                   <li>Spark version: <a href="https://spark.apache.org/downloads.html">SUBST_SPARK_MAJOR_VERSION_WITH_EXCLUDES</a> (<a href="https://spark.apache.org/docs/SUBST_SPARK_VERSION/">documentation</a>)</li>
-                  <li>Sparkling Water: <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/index.html">Documentation</a> <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/CHANGELOG.html">Changelog</a></li>
+                  <li>Sparkling Water: <a href="doc/index.html">Documentation</a> <a href="doc/CHANGELOG.html">Changelog</a></li>
                   <li>Release date: SUBST_BUILD_TIME_LOCAL</li>
                 </ul>
               </div>
@@ -752,7 +752,7 @@ <h2>Integration info</h2>
                 <ul>
                   <li>H2O version: <a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/index.html">SUBST_H2O_VERSION.SUBST_H2O_BUILD SUBST_H2O_NAME</a> (<a href="https://h2o-release.s3.amazonaws.com/h2o/SUBST_H2O_BRANCH_NAME/SUBST_H2O_BUILD/docs-website/h2o-docs/index.html">documentation</a>)</li>
                   <li>Spark version: <a href="https://spark.apache.org/downloads.html">SUBST_SPARK_MAJOR_VERSION_WITH_EXCLUDES</a> (<a href="https://spark.apache.org/docs/SUBST_SPARK_VERSION/">documentation</a>)</li>
-                  <li>Sparkling Water: <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/index.html">Documentation</a> <a href="https://h2o-release.s3.amazonaws.com/SUBST_S3_BUCKET/doc/CHANGELOG.html">Changelog</a></li>
+                  <li>Sparkling Water: <a href="doc/index.html">Documentation</a> <a href="doc/CHANGELOG.html">Changelog</a></li>
                   <li>Release date: SUBST_BUILD_TIME_LOCAL</li>
                 </ul>
               </div>

diff --git a/gradle-spark2.1.properties b/gradle-spark2.1.properties
@@ -1,5 +1,5 @@
 # Sparkling Water Version
-version=2.1.99999-SNAPSHOT
+version=3.26.1-SNAPSHOT-2.1
 sparkVersion=2.1.3
 minSupportedJavaVersion=1.7
 supportedEmrVersion=emr-5.7.0

diff --git a/gradle-spark2.2.properties b/gradle-spark2.2.properties
@@ -1,5 +1,5 @@
 # Sparkling Water Version
-version=2.2.99999-SNAPSHOT
+version=3.26.1-SNAPSHOT-2.2
 sparkVersion=2.2.3
 minSupportedJavaVersion=1.8
 supportedEmrVersion=emr-5.12.0

diff --git a/gradle-spark2.3.properties b/gradle-spark2.3.properties
@@ -1,5 +1,5 @@
 # Sparkling Water Version
-version=2.3.99999-SNAPSHOT
+version=3.26.1-SNAPSHOT-2.3
 sparkVersion=2.3.3
 minSupportedJavaVersion=1.8
 supportedEmrVersion=emr-5.19.0

diff --git a/gradle-spark2.4.properties b/gradle-spark2.4.properties
@@ -1,5 +1,5 @@
 # Sparkling Water Version
-version=2.4.99999-SNAPSHOT
+version=3.26.1-SNAPSHOT-2.4
 sparkVersion=2.4.3
 minSupportedJavaVersion=1.8
 supportedEmrVersion=emr-5.21.0

diff --git a/gradle/utils.gradle b/gradle/utils.gradle
@@ -61,14 +61,6 @@ def getSupportedHadoopDistributions(h2oMajorName, h2oBuild) {
     return distributions.join(" ")
 }
 
-def getCurrentRSparklingVersion(def projectDir) {
-    def proc = ['R', '-e', "source('for_release/utils.R'); getCurrentVersion(\"src\");"].execute([], file(projectDir))
-    def lines = proc.text.readLines()
-    def idx = lines.findIndexOf { it.contains("[1]") }
-    def patchVersion = lines[idx].replace("[1] ", "").toString()
-    return "0.2." + patchVersion
-}
-
 def getSparkSpecificSourceDir(sparkMajorVersion) {
     if (file("src/main/scala_spark_$sparkMajorVersion").exists()) {
         return ["src/main/scala_spark_$sparkMajorVersion"]
@@ -89,27 +81,37 @@ def getSparkSpecificResourceDir(sparkMajorVersion) {
     }
 }
 
-def getPatchVersionNumber() {
-    def patchVersion = version.toString().split("\\.")[2]
+def getS3BucketPrefix() {
+    def prefix = "sparkling-water/spark-${spark}"
 
+    def branch = getGitBranch()
+    def formattedSWBranch = branch.replaceAll("/", "-")
+    def path
+    if (project.hasProperty("buildAgainstH2OBranch")) {
+        def h2oBranch = project.property("buildAgainstH2OBranch")
+        def formattedH2OBranch = h2oBranch.replaceAll("/", "-")
+        path = "${prefix}/${formattedSWBranch}_${formattedH2OBranch}"
+    } else {
+        def majorVersion = "${version.split("\\.")[0]}.${version.split("\\.")[1]}"
+        def relBranch = "rel-${majorVersion}"
+        if (branch == relBranch) {
+            path = "${prefix}"
+        } else {
+            path = "${prefix}/${formattedSWBranch}"
+        }
+    }
     if (isNightlyBuild) {
-        return patchVersion.split("-")[1]
+        return "${path}/nightly"
     } else {
-        return patchVersion
+        return path
     }
 }
 
 def getS3Bucket() {
-    def prefix = "sparkling-water/${getGitBranch().replaceAll("/", "-")}-${spark}"
-    def suffix = getPatchVersionNumber()
+    def prefix = getS3BucketPrefix()
+    return "${prefix}/${version}"
 
-    if (isNightlyBuild) {
-        return "${prefix}/nightly/${suffix}"
-    } else {
-        return "${prefix}/${suffix}"
-    }
 }
-
 // Export methods by turning them into closures
 ext {
     getOsSpecificCommandLine = this.&getOsSpecificCommandLine
@@ -120,8 +122,8 @@ ext {
     getGitBranch = this.&getGitBranch
     getH2OBranch = this.&getH2OBranch
     getSupportedHadoopDistributions = this.&getSupportedHadoopDistributions
-    getCurrentRSparklingVersion = this.&getCurrentRSparklingVersion
     getSparkSpecificSourceDir = this.&getSparkSpecificSourceDir
     getSparkSpecificResourceDir = this.&getSparkSpecificResourceDir
     getS3Bucket = this.&getS3Bucket
+    getS3BucketPrefix = this.&getS3BucketPrefix
 }
diff --git a/jenkins/Jenkinsfile b/jenkins/Jenkinsfile
@@ -45,6 +45,7 @@ node("docker") {
     sparkVersions = versionLine.split("=")[1].split(" ")
 }
 
+
 pipeline(params) { p ->
     sparkMajorVersions = sparkVersions
     runUnitTests = "${p.runUnitTests}"

diff --git a/jenkins/Jenkinsfile-nightly b/jenkins/Jenkinsfile-nightly
@@ -10,7 +10,7 @@ properties(
 
 def pipeline
 def sparkVersions
-node("docker") {
+node("mr-0xc10") {
     cleanWs()
     checkout scm
     pipeline = load 'jenkins/sparklingWaterPipeline.groovy'