Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SW-690] Log Sparkling Water version during startup of Sparkling Water #549

Merged
merged 3 commits into from
Jan 26, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# generated resource file containing Spark version for which Sparkling Water was built
/core/src/main/resources/spark.version
/core/src/main/resources/h2o.version
/core/src/main/resources/sw.version
*.class

# Mobile Tools for Java (J2ME)
Expand Down
1 change: 1 addition & 0 deletions bin/sparkling-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ cat <<EOF
Spark master (MASTER) : $MASTER
Spark home (SPARK_HOME) : $SPARK_HOME
H2O build version : ${H2O_VERSION}.${H2O_BUILD} ($H2O_NAME)
Sparkling Water version : ${VERSION}
Spark build version : ${SPARK_VERSION}
Scala version : ${SCALA_VERSION}
----
Expand Down
9 changes: 9 additions & 0 deletions core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,18 @@ task createH2OVersionFile {
}
}

task createSparklingWaterVersionFile {
doLast {
File version_file = file("src/main/resources/sw.version")
// Create parent directories if not created yet
version_file.getParentFile().mkdirs()
version_file.write(version)
}
}
test.dependsOn testJar
processResources.dependsOn createSparkVersionFile
processResources.dependsOn createH2OVersionFile
processResources.dependsOn createSparklingWaterVersionFile

integTest {
// Pass references to libraries to test launcher
Expand Down
16 changes: 16 additions & 0 deletions core/src/main/scala/org/apache/spark/h2o/BuildInfo.scala
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,22 @@ object BuildInfo {
case x: Exception => throw new WrongSparkVersion(s"Failed to read H2O version from $VERSION_FILE: ${x.getMessage}")
}
}
}

/**
* Returns Sparkling Water version
*/
def SWVersion = {
val VERSION_FILE: String = "/sw.version"
val stream = getClass.getResourceAsStream(VERSION_FILE)

stream match {
case null => throw new RuntimeException(s"Unknown Sparkling Water version: $VERSION_FILE missing")
case s => try {
scala.io.Source.fromInputStream(s).mkString
} catch {
case x: Exception => throw new WrongSparkVersion(s"Failed to read Sparkling Water version from $VERSION_FILE: ${x.getMessage}")
}
}
}
}
8 changes: 6 additions & 2 deletions core/src/main/scala/org/apache/spark/h2o/H2OContext.scala
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package org.apache.spark.h2o
import java.util.concurrent.atomic.AtomicReference

import org.apache.spark._
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.h2o.backends.SparklingBackend
import org.apache.spark.h2o.backends.external.ExternalH2OBackend
import org.apache.spark.h2o.backends.internal.InternalH2OBackend
Expand Down Expand Up @@ -96,8 +97,11 @@ class H2OContext private(val sparkSession: SparkSession, conf: H2OConf) extends
def init(): H2OContext = {

// Use H2O's logging as H2O info log level is default
Log.info("The following Spark configuration is used: " + _conf.getAll.mkString(", "))

Log.info("Sparkling Water version: " + BuildInfo.SWVersion)
Log.info("Spark version: " + sparkContext.version)
Log.info("Integrated H2O version: " + BuildInfo.H2OVersion)
Log.info("The following Spark configuration is used: \n " + _conf.getAll.mkString("\n "))
Log.info("")
if (!isRunningOnCorrectSpark(sparkContext)) {
throw new WrongSparkVersion(s"You are trying to use Sparkling Water built for Spark ${BuildInfo.buildSparkMajorVersion}," +
s" but your $$SPARK_HOME(=${sparkContext.getSparkHome().getOrElse("SPARK_HOME is not defined!")}) property" +
Expand Down