From 5271cf83683d410f0dfff2f5ce9dab4dd2dcd339 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Sat, 7 Sep 2024 22:04:58 +0200 Subject: [PATCH 1/2] SSL context from file --- .../sql/arangodb/commons/ArangoDBConf.scala | 61 +++++++++++++----- integration-tests/src/test/resources/cert.p12 | Bin 0 -> 2659 bytes .../sql/arangodb/datasource/SslTest.scala | 21 +++++- 3 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 integration-tests/src/test/resources/cert.p12 diff --git a/arangodb-spark-commons/src/main/scala/org/apache/spark/sql/arangodb/commons/ArangoDBConf.scala b/arangodb-spark-commons/src/main/scala/org/apache/spark/sql/arangodb/commons/ArangoDBConf.scala index ca36836a..3fb18d74 100644 --- a/arangodb-spark-commons/src/main/scala/org/apache/spark/sql/arangodb/commons/ArangoDBConf.scala +++ b/arangodb-spark-commons/src/main/scala/org/apache/spark/sql/arangodb/commons/ArangoDBConf.scala @@ -1,19 +1,20 @@ package org.apache.spark.sql.arangodb.commons -import com.arangodb.{ArangoDB, entity} import com.arangodb.model.OverwriteMode +import com.arangodb.{ArangoDB, entity} import org.apache.spark.internal.Logging import org.apache.spark.internal.config._ import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DropMalformedMode, FailFastMode, ParseMode, PermissiveMode} -import java.io.ByteArrayInputStream +import java.io.{ByteArrayInputStream, FileInputStream} import java.security.KeyStore import java.security.cert.CertificateFactory import java.util import java.util.Base64 import javax.net.ssl.{SSLContext, TrustManagerFactory} import scala.collection.JavaConverters.{mapAsJavaMapConverter, mapAsScalaMapConverter} +import scala.util.Using object ArangoDBConf { @@ -69,11 +70,23 @@ object ArangoDBConf { .createWithDefault(false) val SSL_VERIFY_HOST = "ssl.verifyHost" - val verifyHostConf: ConfigEntry[Boolean] = ConfigBuilder(SSL_VERIFY_HOST) + val sslVerifyHostConf: ConfigEntry[Boolean] = ConfigBuilder(SSL_VERIFY_HOST) .doc("hostname verification") .booleanConf .createWithDefault(true) + val SSL_TRUST_STORE_PASSWORD = "ssl.trustStore.password" + val sslTrustStorePasswordConf: OptionalConfigEntry[String] = ConfigBuilder(SSL_TRUST_STORE_PASSWORD) + .doc("trustStore password") + .stringConf + .createOptional + + val SSL_TRUST_STORE_PATH = "ssl.trustStore.path" + val sslTrustStorePathConf: OptionalConfigEntry[String] = ConfigBuilder(SSL_TRUST_STORE_PATH) + .doc("trustStore path") + .stringConf + .createOptional + val SSL_CERT_VALUE = "ssl.cert.value" val sslCertValueConf: OptionalConfigEntry[String] = ConfigBuilder(SSL_CERT_VALUE) .doc("base64 encoded certificate") @@ -100,7 +113,7 @@ object ArangoDBConf { val SSL_KEYSTORE_TYPE = "ssl.keystore.type" val sslKeystoreTypeConf: ConfigEntry[String] = ConfigBuilder(SSL_KEYSTORE_TYPE) - .doc("keystore type") + .doc("keystore type, deprecated: use ssl.trustStore.type instead") .stringConf .createWithDefault("jks") @@ -268,7 +281,9 @@ object ArangoDBConf { CONTENT_TYPE -> contentTypeConf, TIMEOUT -> timeoutConf, SSL_ENABLED -> sslEnabledConf, - SSL_VERIFY_HOST -> verifyHostConf, + SSL_VERIFY_HOST -> sslVerifyHostConf, + SSL_TRUST_STORE_PASSWORD -> sslTrustStorePasswordConf, + SSL_TRUST_STORE_PATH -> sslTrustStorePathConf, SSL_CERT_VALUE -> sslCertValueConf, SSL_CERT_TYPE -> sslCertTypeConf, SSL_CERT_ALIAS -> sslCertAliasConf, @@ -479,7 +494,11 @@ class ArangoDBDriverConf(opts: Map[String, String]) extends ArangoDBConf(opts) { val sslEnabled: Boolean = getConf(sslEnabledConf) - val verifyHost: Boolean = getConf(verifyHostConf) + val verifyHost: Boolean = getConf(sslVerifyHostConf) + + val sslTrustStorePassword: Option[String] = getConf(sslTrustStorePasswordConf) + + val sslTrustStorePath: Option[String] = getConf(sslTrustStorePathConf) val sslCertValue: Option[String] = getConf(sslCertValueConf) @@ -489,6 +508,7 @@ class ArangoDBDriverConf(opts: Map[String, String]) extends ArangoDBConf(opts) { val sslAlgorithm: String = getConf(sslAlgorithmConf) + // FIXME: merge with sslTrustStoreType val sslKeystoreType: String = getConf(sslKeystoreTypeConf) val sslProtocol: String = getConf(sslProtocolConf) @@ -514,21 +534,32 @@ class ArangoDBDriverConf(opts: Map[String, String]) extends ArangoDBConf(opts) { builder } - def getSslContext: SSLContext = sslCertValue match { - case Some(b64cert) => - val is = new ByteArrayInputStream(Base64.getDecoder.decode(b64cert)) + def getSslContext: SSLContext = { + if (sslCertValue.isDefined) { + val is = new ByteArrayInputStream(Base64.getDecoder.decode(sslCertValue.get)) val cert = CertificateFactory.getInstance(sslCertType).generateCertificate(is) val ks = KeyStore.getInstance(sslKeystoreType) ks.load(null) // scalastyle:ignore null ks.setCertificateEntry(sslCertAlias, cert) - val tmf = TrustManagerFactory.getInstance(sslAlgorithm) - tmf.init(ks) - val sc = SSLContext.getInstance(sslProtocol) - sc.init(null, tmf.getTrustManagers, null) // scalastyle:ignore null - sc - case None => SSLContext.getDefault + createSslContext(ks) + } else if (sslTrustStorePath.isDefined) { + val ks = KeyStore.getInstance(sslKeystoreType) + Using(new FileInputStream(sslTrustStorePath.get)) { is => + ks.load(is, sslTrustStorePassword.map(_.toCharArray).orNull) + } + createSslContext(ks) + } else { + SSLContext.getDefault + } } + private def createSslContext(ks: KeyStore): SSLContext = { + val tmf = TrustManagerFactory.getInstance(sslAlgorithm) + tmf.init(ks) + val sc = SSLContext.getInstance(sslProtocol) + sc.init(null, tmf.getTrustManagers, null) // scalastyle:ignore null + sc + } } diff --git a/integration-tests/src/test/resources/cert.p12 b/integration-tests/src/test/resources/cert.p12 new file mode 100644 index 0000000000000000000000000000000000000000..1e9fa4f84a01577b0dd1118635faaedc91a49db4 GIT binary patch literal 2659 zcmai$XEYlM8^Tav3I0q)QX~N?@eozs8K`_waP6eW=koxsXd}Xi_1mZs8zAm z9^u-xV%Dg2Z|`~E)BEv$c+PYF|KEq_+wVLGECdAvP$IAp7HY6)jBd;^J%9#KfQ3+l zun-^u3%*2PLFj)GbpaM+_p`bO1W^23rvDNE0`)6U(IdDQ&;{U@ zB>pevPd_xrg$81cLMA8enY$bWX6hmgciV*9(t{7YQ&EIn6WY-xj|-pgvtRVSR?8G@ z>E)~DtL{AeU|$Xk5VC7m=VWoymi(yaZ8M#(kuC;}z@*u&RF9ZmftOH^m#On5W{~ni ztxwZk#9LEqN5~RY`uYjztB3Vh0-QUD)P8a#?P;wk%XQs_{uk^vo=e#PZl?|d_TbP! zG05ngv9PUjEq}_&O~#TE|A#yLgKg(DoYM2F>bOcI+g;OH2mdJY7!b zi@K{2>NZya3HGv7W>TF#Sl7Hj&kY0auN|K*Cv7OKBj7Vb*UYtm_i0C3{~)*~H<3&3 zEQr=lB@l`ZouwpbzD5o9Zb)IOaK_J*IPi>HfY>_=U%kg*BHsaQu6YKeWpuzq=ehSa z6*qIq_Mw_g%3U+2b%A^F2Hh@&@8LyI^I}x2*u=+g1s?*!l*-(Y+^>z4ttMAq*<8sE z2d}92S$UZdifyl=yQ3P0b)@{X<=-ZX$kX=jTOSsQU02fgETUIK*LQU8SLLq_G!1Xx z0;ldUZdMHn+BL-mwef`6t92y%wJYO@bcgL3vCj>slg`-dsR}0<`zeY3G>5ZF+&g4( zq}5I@<`c(>ilqSGs5~i)$x3kDUEFL>WEEMu$`N7Qm+{xrgx>MfU>1w(ji@elv-v<` zpq=q;*{aBf1rMvoM>Ps%aU&$nATUA1k)Dn-C$I3ld*=AY*jUSr`U84XkyH}N&AsYMMPC7Qu|!NZ`QJEDRpt_W3`RPItgBo#qt_wp?~T!R!7!mKZd z&_df-6PgiNs)S$O_(w!kp&%?(@XtEn=lD`z`M;y+AOPS`sQ9y#`!Df;rI-T?z>7C} zH$L=F*)LrGPw~{d5T5{gpb;dy-ZbruQEw0yG-OsfJfP9KE0Mmp%F{35mGf)^5zd0c93L84p5@TMz>w8n1%6yGP#O?!8c`UknHSn6>!IS;7WL^>L51 z4oriT7F5W-rLRHUDA6{=W(}1r2zZqM*%+yr(s1FJ29UpkW4|PzJpwr;}>1n z=TYu7{!rTGHc6uh5ZdxiR)5vHUHNp)(d&qNwHamQ-7C>;D?OUXs3;hxPjY_6T9w5L zO&F|Yj+Z^lA=>4(#_Xc*bDKiiEtC!t{V2WU0-htDqm3?OKs$Ujj8ZU@OBHAma@E7$ zc4{d(@?3`>roUY6R8sHeWmA7>GKHZ(tgPmV4$A)Kf#=&3lX5+ZdOt@iih9TwNTT_I>2ur05Ss*ZjB+J3Z0Uu;$= z9p~ya=pEPH7z$2;G%O@eE)u$=t6Y2mdLq8@h^#sAIVG+q_jLnn*k%fO-!e!+$!ohk zABib#2?n0W5wQhkCqVA6BAV1X`r zNi z6rav7*8ut{%2&}iF_)AI2e*71fwgzgXE(vPuC*CW8}u?6$}sH!Qv5N7cCW5&R??VT4gAe2YO5 z4fjfVYzIu7GB>Bg^;j$-T}4VQlMcjztuwS6d4W3RxM{JGz1nnwhg& zqP;Gi5wz*>HUJjpOG?hJPb^_n2T9;uMeU&$p=*i@C zE46!w^D2(R+S?pjcc>g{-2U=}f9(J_iM+Vacj;in5KYa}qVKJSKBajlBXHkP?D;;* zbPU%ASy=QnN8aLYnv;4KfRWA{Yjsp{L>#UDX#X}=`;N?k;VeHZo!{}5i8MJkX`);@ z8PM1bI5Vi3zx-4Y`xk_0YDM-Ya>whzuB^m)n9N}afrzE3V&S(+l~Ud z6_d#m)v>(-qB-){_(rznU?(O=d6pV`!<28m)GQ{-lj;@3t~M8C@~@_ixaSK4vUoJ( z9uHD%dok4~Ye)+t1b>MZ%p%PSVwNaYk98sczLePA7y9QkWv&0)3phdzA&sE_^_Bqv zR1_e-;9Ck|Zey5B?f3(iQ4;j`c{{5fvzL@EFT+l1h2gWQAX?;jHihuoChi~A{tbjx B protocol, - ArangoDBConf.COLLECTION -> "sslTest" + ArangoDBConf.COLLECTION -> "sslTest", + ArangoDBConf.SSL_CERT_VALUE -> SslTest.b64cert, )) .load() df.show() } + @ParameterizedTest + @ValueSource(strings = Array("vst", "http", "http2")) + def sslTrustStorePathTest(protocol: String): Unit = { + assumeTrue(protocol != "vst" || isLessThanVersion(version.getVersion, 3, 12, 0)) + val df = spark.read + .format(classOf[DefaultSource].getName) + .options(options ++ Map( + ArangoDBConf.PROTOCOL -> protocol, + ArangoDBConf.COLLECTION -> "sslTest", + ArangoDBConf.SSL_TRUST_STORE_PATH -> "src/test/resources/cert.p12", + ArangoDBConf.SSL_TRUST_STORE_PASSWORD -> "12345678" + )) + .load() + df.show() + } } object SslTest { @@ -101,7 +117,6 @@ object SslTest { private val options = Map( ArangoDBConf.SSL_ENABLED -> "true", ArangoDBConf.SSL_VERIFY_HOST -> "false", - ArangoDBConf.SSL_CERT_VALUE -> b64cert, ArangoDBConf.DB -> database, ArangoDBConf.USER -> user, ArangoDBConf.PASSWORD -> password, From c25606e5dd373e7aa8e0eee32fad223fa9a48ed7 Mon Sep 17 00:00:00 2001 From: Michele Rastelli Date: Sat, 7 Sep 2024 22:25:03 +0200 Subject: [PATCH 2/2] fix Scala 2.12 --- .../apache/spark/sql/arangodb/commons/ArangoDBConf.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arangodb-spark-commons/src/main/scala/org/apache/spark/sql/arangodb/commons/ArangoDBConf.scala b/arangodb-spark-commons/src/main/scala/org/apache/spark/sql/arangodb/commons/ArangoDBConf.scala index 3fb18d74..7f538c0e 100644 --- a/arangodb-spark-commons/src/main/scala/org/apache/spark/sql/arangodb/commons/ArangoDBConf.scala +++ b/arangodb-spark-commons/src/main/scala/org/apache/spark/sql/arangodb/commons/ArangoDBConf.scala @@ -14,7 +14,6 @@ import java.util import java.util.Base64 import javax.net.ssl.{SSLContext, TrustManagerFactory} import scala.collection.JavaConverters.{mapAsJavaMapConverter, mapAsScalaMapConverter} -import scala.util.Using object ArangoDBConf { @@ -544,8 +543,11 @@ class ArangoDBDriverConf(opts: Map[String, String]) extends ArangoDBConf(opts) { createSslContext(ks) } else if (sslTrustStorePath.isDefined) { val ks = KeyStore.getInstance(sslKeystoreType) - Using(new FileInputStream(sslTrustStorePath.get)) { is => + val is = new FileInputStream(sslTrustStorePath.get) + try { ks.load(is, sslTrustStorePassword.map(_.toCharArray).orNull) + } finally { + is.close() } createSslContext(ks) } else {