[SW-1708] Simplify distribution of security files (#1614)

h2oai · Nov 13, 2019 · 994c99c · 994c99c
1 parent 485d98c
commit 994c99c
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 12 deletions.
diff --git a/core/src/main/scala/org/apache/spark/h2o/backends/SharedBackendConf.scala b/core/src/main/scala/org/apache/spark/h2o/backends/SharedBackendConf.scala
@@ -40,14 +40,12 @@ trait SharedBackendConf {
   def isSparkVersionCheckEnabled = sparkConf.getBoolean(PROP_SPARK_VERSION_CHECK_ENABLED._1, PROP_SPARK_VERSION_CHECK_ENABLED._2)
   def isFailOnUnsupportedSparkParamEnabled = sparkConf.getBoolean(PROP_FAIL_ON_UNSUPPORTED_SPARK_PARAM._1, PROP_FAIL_ON_UNSUPPORTED_SPARK_PARAM._2)
   def jks = sparkConf.getOption(PROP_JKS._1)
-  private[backends] def jksDistributed = sparkConf.getOption(PROP_JKS._3)
   def jksPass = sparkConf.getOption(PROP_JKS_PASS._1)
   def jksAlias = sparkConf.getOption(PROP_JKS_ALIAS._1)
   def hashLogin = sparkConf.getBoolean(PROP_HASH_LOGIN._1, PROP_HASH_LOGIN._2)
   def ldapLogin = sparkConf.getBoolean(PROP_LDAP_LOGIN._1, PROP_LDAP_LOGIN._2)
   def kerberosLogin = sparkConf.getBoolean(PROP_KERBEROS_LOGIN._1, PROP_KERBEROS_LOGIN._2)
   def loginConf = sparkConf.getOption(PROP_LOGIN_CONF._1)
-  private[backends] def loginConfDistributed = sparkConf.getOption(PROP_LOGIN_CONF._3)
   def userName = sparkConf.getOption(PROP_USER_NAME._1)
   def sslConf = sparkConf.getOption(PROP_SSL_CONF._1)
   def autoFlowSsl = sparkConf.getBoolean(PROP_AUTO_SSL_FLOW._1, PROP_AUTO_SSL_FLOW._2)
@@ -242,7 +240,7 @@ object SharedBackendConf {
   val PROP_FAIL_ON_UNSUPPORTED_SPARK_PARAM = ("spark.ext.h2o.fail.on.unsupported.spark.param", true)
 
   /** Path to Java KeyStore file. */
-  val PROP_JKS = ("spark.ext.h2o.jks", None, "spark.ext.h2o.jks.distributed")
+  val PROP_JKS = ("spark.ext.h2o.jks", None)
 
   /** Password for Java KeyStore file. */
   val PROP_JKS_PASS = ("spark.ext.h2o.jks.pass", None)
@@ -260,7 +258,7 @@ object SharedBackendConf {
   val PROP_KERBEROS_LOGIN = ("spark.ext.h2o.kerberos.login", false)
 
   /** Login configuration file. */
-  val PROP_LOGIN_CONF = ("spark.ext.h2o.login.conf", None, "spark.ext.h2o.login.conf.distributed")
+  val PROP_LOGIN_CONF = ("spark.ext.h2o.login.conf", None)
 
   /** Override user name for cluster. */
   val PROP_USER_NAME = ("spark.ext.h2o.user.name", None)

diff --git a/core/src/main/scala/org/apache/spark/h2o/backends/SharedBackendUtils.scala b/core/src/main/scala/org/apache/spark/h2o/backends/SharedBackendUtils.scala
@@ -109,26 +109,28 @@ private[backends] trait SharedBackendUtils extends Logging with Serializable {
     for(fileProperty <- conf.getFileProperties()) {
       for (filePath <- conf.getOption(fileProperty._1)) {
         sc.addFile(filePath)
-        val distributedLocalPath = SparkFiles.get(new File(filePath).getName)
-        conf.set(fileProperty._3, distributedLocalPath)
       }
     }
   }
 
+  private def getDistributedFilePath(fileConf: Option[String]): Option[String] = {
+    fileConf.map(name => SparkFiles.get(new File(name).getName))
+  }
+
   def defaultLogDir(appId: String): String = {
     System.getProperty("user.dir") + java.io.File.separator + "h2ologs" + File.separator + appId
   }
 
-  def getH2OSecurityArgs(conf: H2OConf, internalWorker: Boolean): Seq[String] = {
+  def getH2OSecurityArgs(conf: H2OConf): Seq[String] = {
     new ArgumentBuilder()
-      .add("-jks", if (internalWorker) conf.jks.map(SparkFiles.get) else conf.jksDistributed)
+      .add("-jks", getDistributedFilePath(conf.jks))
       .add("-jks_pass", conf.jksPass)
       .add("-jks_alias", conf.jksAlias)
       .addIf("-hash_login", conf.hashLogin)
       .addIf("-ldap_login", conf.ldapLogin)
       .addIf("-kerberos_login", conf.kerberosLogin)
       .add("-user_name", conf.userName)
-      .add("-login_conf", if (internalWorker) conf.loginConf.map(SparkFiles.get) else conf.loginConfDistributed)
+      .add("-login_conf", getDistributedFilePath(conf.loginConf))
       .buildArgs()
   }
 
@@ -152,7 +154,7 @@ private[backends] trait SharedBackendUtils extends Logging with Serializable {
   def getH2OWorkerAsClientArgs(conf: H2OConf): Seq[String] = {
     new ArgumentBuilder()
       .add(getH2OCommonArgs(conf))
-      .add(getH2OSecurityArgs(conf, false))
+      .add(getH2OSecurityArgs(conf))
       .addIf("-quiet", !conf.clientVerboseOutput)
       .add("-log_level", conf.h2oClientLogLevel)
       .add("-log_dir", conf.h2oClientLogDir)

diff --git a/core/src/main/scala/org/apache/spark/h2o/backends/external/ExternalH2OBackend.scala b/core/src/main/scala/org/apache/spark/h2o/backends/external/ExternalH2OBackend.scala
@@ -113,7 +113,7 @@ class ExternalH2OBackend(val hc: H2OContext) extends SparklingBackend with Exter
       cmdToLaunch = cmdToLaunch ++ Seq("-network", hc.getConf.nodeNetworkMask.get)
     }
 
-    cmdToLaunch = cmdToLaunch ++ getH2OSecurityArgs(hc.getConf, false)
+    cmdToLaunch = cmdToLaunch ++ getH2OSecurityArgs(hc.getConf)
 
     if (hc.getConf.kerberosKeytab.isDefined && hc.getConf.kerberosPrincipal.isDefined) {
       cmdToLaunch = cmdToLaunch ++ Seq("-principal",

diff --git a/core/src/main/scala/org/apache/spark/h2o/backends/internal/InternalBackendUtils.scala b/core/src/main/scala/org/apache/spark/h2o/backends/internal/InternalBackendUtils.scala
@@ -86,7 +86,7 @@ object InternalBackendUtils extends InternalBackendUtils {
 
     new ArgumentBuilder()
       .add(getH2OCommonArgs(conf))
-      .add(getH2OSecurityArgs(conf, true))
+      .add(getH2OSecurityArgs(conf))
       .add("-log_level", conf.h2oNodeLogLevel)
       .add("-baseport", conf.nodeBasePort)
       .add("-log_dir", InternalBackendUtils.getH2ONodeLogDir(conf, SparkEnv.get))