From b7265335995cb0c4ba42c2027d00f19da8af6cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20H=C3=A1va?= Date: Mon, 4 Nov 2019 10:56:50 +0100 Subject: [PATCH] [SW-1681] Change sw version to include also patch within one h2o version (#1585) (cherry picked from commit 562c71f5a80f1b2a35c88c3ae27e21d1e1eecd26) --- doc/build.gradle | 6 --- .../site/sphinx/deployment/sw_azure_dbc.rst | 2 +- doc/src/site/sphinx/migration_guide.rst | 13 ++++++- gradle.properties | 2 +- gradle/release.gradle | 1 - jenkins/Jenkinsfile-release | 22 ++++++++--- jenkins/sparklingWaterPipeline.groovy | 37 ++++++++++++------- py/build.gradle | 4 +- py/src/ai/h2o/sparkling/VersionComponents.py | 7 ++-- .../unit/simple/test_version_components.py | 14 ++++--- 10 files changed, 69 insertions(+), 39 deletions(-) diff --git a/doc/build.gradle b/doc/build.gradle index aca000733d..a6d48a472b 100644 --- a/doc/build.gradle +++ b/doc/build.gradle @@ -25,17 +25,11 @@ task substitute() { doLast { def siteDir = "${buildDir}/site" - def swOnlyVersion = version.tokenize("-")[0] - def (first, second, minorVersion) = swOnlyVersion.tokenize(".") - def majorVersion = "${first}.${second}" - new File(siteDir).eachFileRecurse(FILES) { if (it.name.endsWith('.html')) { def contents = file(it).getText('UTF-8') contents = contents .replaceAll("SUBST_SW_VERSION", version) - .replaceAll("SUBST_SW_MAJOR_VERSION", majorVersion) - .replaceAll("SUBST_SW_MINOR_VERSION", minorVersion) .replaceAll("SUBST_SPARK_VERSION", sparkVersion) .replaceAll("SUBST_SPARK_MAJOR_VERSION", sparkMajorVersion) .replaceAll("SUBST_H2O_VERSION", h2oVersion) diff --git a/doc/src/site/sphinx/deployment/sw_azure_dbc.rst b/doc/src/site/sphinx/deployment/sw_azure_dbc.rst index ae3a5ffcc0..5ecebff75c 100644 --- a/doc/src/site/sphinx/deployment/sw_azure_dbc.rst +++ b/doc/src/site/sphinx/deployment/sw_azure_dbc.rst @@ -19,7 +19,7 @@ To start Sparkling Water ``H2OContext`` on Databricks Azure, the steps are: 3. Add Sparkling Water dependency - In order to create the Java library in Databricks, go to **Libraries**, select **Maven** as the library source and type the following into the coordinates field: ``sparkling-water-package_2.11:SUBST_SW_MINOR_VERSION``. + In order to create the Java library in Databricks, go to **Libraries**, select **Maven** as the library source and type the following into the coordinates field: ``ai.h2o:sparkling-water-package_2.11:SUBST_SW_VERSION``. .. figure:: ../images/databricks_sw_maven.png :alt: Uploading Sparkling Water assembly JAR diff --git a/doc/src/site/sphinx/migration_guide.rst b/doc/src/site/sphinx/migration_guide.rst index 3a99dbc402..2397bff9ba 100644 --- a/doc/src/site/sphinx/migration_guide.rst +++ b/doc/src/site/sphinx/migration_guide.rst @@ -89,4 +89,15 @@ Removal of Deprecated Methods and Classes - On PySparkling ``H2OGLM`` API, we removed deprecated parameter ``alpha`` in favor of ``alphaValue`` and ``lambda_`` in favor of ``lambdaValue``. On Both PySparkling and Sparkling Water ``H2OGLM`` API, we removed methods ``getAlpha`` in favor of ``getAlphaValue``, ``getLambda`` in favor of ``getLambdaValue``, ``setAlpha`` in favor of ``setAlphaValue`` and - ``setLambda`` in favor of ``setLambdaValue``. These changes ensure the consistency across Python and Scala APIs. \ No newline at end of file + ``setLambda`` in favor of ``setLambdaValue``. These changes ensure the consistency across Python and Scala APIs. + +Change of Versioning Scheme +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Version of Sparkling Water is changed to the following pattern: ``H2OVersion-SWPatchVersion-SparkVersion``, where: +``H2OVersion`` is full H2O Version which is integrated to Sparkling Water. ``SWPatchVersion`` is used to specify +a patch version and ``SparkVersion`` is a Spark version. This change of scheme allows us to do releases of Sparkling Water +without the need of releasing H2O if there is only change on the Sparkling Water side. In that case, we just increment the +``SWPatchVersion``. The new version therefore looks, for example, like ``3.26.0.9-2-2.4``. This version tells us this +Sparkling Water is integrating H2O ``3.26.0.9``, it is the second release with ``3.26.0.9`` version and is for Spark ``2.4``. + diff --git a/gradle.properties b/gradle.properties index bb9d2fef05..e3eb1a5e85 100644 --- a/gradle.properties +++ b/gradle.properties @@ -26,4 +26,4 @@ supportedSparkVersions=2.1 2.2 2.3 2.4 # Select for which Spark version is Sparkling Water built by default spark=2.4 # Sparkling Water Version -version=3.28.1-SNAPSHOT +version=3.28.0.1-1-SNAPSHOT diff --git a/gradle/release.gradle b/gradle/release.gradle index e7ac74446c..535dd01f46 100644 --- a/gradle/release.gradle +++ b/gradle/release.gradle @@ -8,7 +8,6 @@ apply plugin: 'net.researchgate.release' release { tagTemplate = 'RELEASE-$version' - // Safe point - do releases only from release branch - can be deleted in future failOnUnversionedFiles = false failOnCommitNeeded = false preCommitText = ":tada: " diff --git a/jenkins/Jenkinsfile-release b/jenkins/Jenkinsfile-release index 3ec369f365..8abe73069d 100644 --- a/jenkins/Jenkinsfile-release +++ b/jenkins/Jenkinsfile-release @@ -5,7 +5,8 @@ properties( [ parameters( - [ booleanParam(name: 'updateChangeLog', defaultValue: true, description: "Update change log"), + [ booleanParam(name: 'wasH2OUpgraded', defaultValue: true, description: "True if H2O was upgraded in this release or not"), + booleanParam(name: 'updateChangeLog', defaultValue: true, description: "Update change log"), string(name: 'releaseFor', defaultValue: 'all', description: "For which Spark Major version" + " do a release. By default, do release for all supported released versions"), booleanParam(name: 'buildConda', defaultValue: true, description: 'Build Conda'), @@ -90,21 +91,32 @@ String getVersionNoSuffix() { String getNextVersionNoSuffix(params) { def majorVersion = getMajorVersion(params) + def minorVersion = getMinorVersion(params) def patchVersion = getPatchVersion(params) - return "${majorVersion}.${patchVersion.toInteger() + 1}-SNAPSHOT" + if (params.wasH2OUpgraded.toBoolean()) { + return "${majorVersion}.${minorVersion.toInteger() + 1}-1-SNAPSHOT" + } else { + return "${majorVersion}.${minorVersion}-${patchVersion.toInteger() + 1}-SNAPSHOT" + } } String getMajorVersion(params) { def v = getVersion(params) - def split = v.split("\\.") - return "${split[0]}.${split[1]}".toString() + def split = v.split("-")[0].split("\\.") + return "${split[0]}.${split[1]}.${split[2]}".toString() +} + +String getMinorVersion(params) { + def v = getVersion(params) + def split = v.split("-")[0].split("\\.") + return "${split[3]}".toString() } String getPatchVersion(params) { def v = getVersion(params) def split = v.split("-") - return "${split[0].split("\\.")[2]}" + return "${split[1]}".toString() } def prepareReleaseNotes(sparkMajorVersions) { diff --git a/jenkins/sparklingWaterPipeline.groovy b/jenkins/sparklingWaterPipeline.groovy index 6872762ccf..3b570a2d9c 100644 --- a/jenkins/sparklingWaterPipeline.groovy +++ b/jenkins/sparklingWaterPipeline.groovy @@ -10,21 +10,32 @@ def getS3Path(config) { String getNightlyVersion(config) { def sparkMajorVersion = config.sparkMajorVersion - def versionLine = readFile("gradle.properties").split("\n").find() { line -> line.startsWith('version') } - def version = versionLine.split("=")[1] - if (config.uploadNightly.toBoolean()) { - def buildNumber - try { - def lastVersion = "https://h2o-release.s3.amazonaws.com/sparkling-water/spark-${config.sparkMajorVersion}/${getS3Path(config)}latest".toURL().getText().toString() - def splits = lastVersion.split("-") - buildNumber = splits[1].toInteger() + 1 - } catch (Exception ignored) { - buildNumber = 1 + def version = readFile("gradle.properties").split("\n").find() { line -> line.startsWith("version") }.split("=")[1] + def versionParts = version.split("-") + def h2oPart = versionParts[0] + def swPatch = versionParts[1] + def swNightlyBuildNumber + try { + def lastVersion = "https://h2o-release.s3.amazonaws.com/sparkling-water/spark-${config.sparkMajorVersion}/${getS3Path(config)}latest".toURL().getText().toString() + def lastVersionParts = lastVersion.split("-") + def lastH2OPart = lastVersionParts[0] + def lastSWPart = lastVersionParts[1] + if (lastSWPart.contains(".")) { + def lastSWParts = lastSWPart.split("\\.") + def lastSWPatch = lastSWParts[0] + def lastSWBuild = lastSWParts[1] + if (lastH2OPart != h2oPart || lastSWPatch != swPatch) { + swNightlyBuildNumber = 1 // reset the nightly build number + } else { + swNightlyBuildNumber = lastSWBuild.toInteger() + 1 + } + } else { + swNightlyBuildNumber = 1 } - return "${version.split("-")[0]}-${buildNumber}-${sparkMajorVersion}" - } else { - return version + } catch (Exception ignored) { + swNightlyBuildNumber = 1 } + return "${h2oPart}-${swPatch}.${swNightlyBuildNumber}-${sparkMajorVersion}" } String getSparkVersion(config) { diff --git a/py/build.gradle b/py/build.gradle index 891fcc9ee2..ada620e869 100644 --- a/py/build.gradle +++ b/py/build.gradle @@ -127,7 +127,7 @@ def copyPySetup() { filter { it.replaceAll("SUBST_SPARK_MAJOR_VERSION", sparkMajorVersion) .replaceAll("SUBST_SPARK_VERSION", sparkVersion) - .replaceAll("SUBST_SW_VERSION", version.substring(0, version.lastIndexOf("-"))) + .replaceAll("SUBST_SW_VERSION", version.substring(0, version.lastIndexOf("-")).replace("-", "_")) .replaceAll("SUBST_SPARK_MAJOR_VERSION", sparkMajorVersion) } into pkgDir @@ -166,7 +166,7 @@ def copyPySetup() { filter { it.replaceAll("SUBST_SPARK_MAJOR_VERSION", sparkMajorVersion) .replaceAll("SUBST_SPARK_VERSION", sparkVersion) - .replaceAll("SUBST_SW_VERSION", version.substring(0, version.lastIndexOf("-"))) + .replaceAll("SUBST_SW_VERSION", version.substring(0, version.lastIndexOf("-")).replace("-", "_")) } into condaDir } diff --git a/py/src/ai/h2o/sparkling/VersionComponents.py b/py/src/ai/h2o/sparkling/VersionComponents.py index ee31c8af28..443cf93c8c 100644 --- a/py/src/ai/h2o/sparkling/VersionComponents.py +++ b/py/src/ai/h2o/sparkling/VersionComponents.py @@ -21,14 +21,15 @@ class VersionComponents(object): @staticmethod def parseFromSparklingWaterVersion(version): - match = re.search(r"^((\d+\.\d+)\.(\d+))(-(\d+))?-(\d+\.\d+)$", version) + match = re.search(r"^((\d+\.\d+\.\d+)\.(\d+)-(\d+))(\.(\d+))?-(\d+\.\d+)$", version) result = VersionComponents() result.fullVersion = match.group(0) result.sparklingVersion = match.group(1) result.sparklingMajorVersion = match.group(2) result.sparklingMinorVersion = match.group(3) - result.nightlyVersion = match.group(5) - result.sparkMajorMinorVersion = match.group(6) + result.sparklingPatchVersion = match.group(4) + result.nightlyVersion = match.group(6) + result.sparkMajorMinorVersion = match.group(7) return result @staticmethod diff --git a/py/tests/unit/simple/test_version_components.py b/py/tests/unit/simple/test_version_components.py index 7d0df40cad..36934bb98b 100644 --- a/py/tests/unit/simple/test_version_components.py +++ b/py/tests/unit/simple/test_version_components.py @@ -19,26 +19,28 @@ def testVersionComponentsCanBeParsedFromRegularVersion(): - version = "3.26.2-2.4" + version = "3.26.0.2-2-2.4" components = VersionComponents.parseFromSparklingWaterVersion(version) assert components.fullVersion == version - assert components.sparklingVersion == "3.26.2" - assert components.sparklingMajorVersion == "3.26" + assert components.sparklingVersion == "3.26.0.2-2" + assert components.sparklingMajorVersion == "3.26.0" assert components.sparklingMinorVersion == "2" + assert components.sparklingPatchVersion == "2" assert components.nightlyVersion is None assert components.sparkMajorMinorVersion == "2.4" def testVersionComponentsCanBeParsedFromNightlyVersion(): - version = "3.28.1-14-2.3" + version = "3.28.0.1-1.14-2.3" components = VersionComponents.parseFromSparklingWaterVersion(version) assert components.fullVersion == version - assert components.sparklingVersion == "3.28.1" - assert components.sparklingMajorVersion == "3.28" + assert components.sparklingVersion == "3.28.0.1-1" + assert components.sparklingMajorVersion == "3.28.0" assert components.sparklingMinorVersion == "1" + assert components.sparklingPatchVersion == "1" assert components.nightlyVersion == "14" assert components.sparkMajorMinorVersion == "2.3"