[SPARK-27831][FOLLOW-UP][SQL][TEST] Should not use maven to add Hive test jars #25690
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -647,3 +647,24 @@ private[sql] class TestHiveSessionStateBuilder( | |
|
|
||
| override protected def newBuilder: NewBuilder = new TestHiveSessionStateBuilder(_, _) | ||
| } | ||
|
|
||
| private[hive] object HiveTestJars { | ||
| private val repository = SQLConf.ADDITIONAL_REMOTE_REPOSITORIES.defaultValueString | ||
|
There was a problem hiding this comment. We can also verify that the default value is valid. |
||
| private val hiveTestJarsDir = Utils.createTempDir() | ||
|
|
||
| def getHiveContribJar(version: String = HiveUtils.builtinHiveVersion): File = | ||
| getJarFromUrl(s"${repository}org/apache/hive/hive-contrib/" + | ||
| s"$version/hive-contrib-$version.jar") | ||
| def getHiveHcatalogCoreJar(version: String = HiveUtils.builtinHiveVersion): File = | ||
| getJarFromUrl(s"${repository}org/apache/hive/hcatalog/hive-hcatalog-core/" + | ||
| s"$version/hive-hcatalog-core-$version.jar") | ||
|
|
||
| private def getJarFromUrl(urlString: String): File = { | ||
| val fileName = urlString.split("/").last | ||
| val targetFile = new File(hiveTestJarsDir, fileName) | ||
| if (!targetFile.exists()) { | ||
| Utils.doFetchFile(urlString, hiveTestJarsDir, fileName, new SparkConf, null, null) | ||
| } | ||
| targetFile | ||
| } | ||
|
There was a problem hiding this comment. cc @srowen @dongjoon-hyun @HyukjinKwon Do you think this is stable? There was a problem hiding this comment. I don't know how stable this way it is. While Sean or Dongjoon might have an idea about this, I would verify if There was a problem hiding this comment. I think this could be OK. I was really thinking of refactoring the logic that directly downloads the JAR, found in If you just want single JARs, how about just constructing the URL for it on There was a problem hiding this comment. Do you prefer this? private val repository = "https://repository.apache.org/content/repositories/releases/"
private val hiveContribUrl = s"${repository}org/apache/hive/hive-contrib/" +
s"${HiveUtils.builtinHiveVersion}/"
private val hiveHcatalogCoreUrl = s"${repository}org/apache/hive/hcatalog/hive-hcatalog-core/" +
s"${HiveUtils.builtinHiveVersion}/"
def getHiveContribJar: File = getFileFromUrl(hiveContribUrl,
s"hive-contrib-${HiveUtils.builtinHiveVersion}.jar")
def getHiveHcatalogCoreJar: File = getFileFromUrl(hiveHcatalogCoreUrl,
s"hive-hcatalog-core-${HiveUtils.builtinHiveVersion}.jar")
private def getFileFromUrl(urlString: String, filename: String): File = {
val hiveTestJars = new File("/tmp/test-spark/hiveTestJars")
if (!hiveTestJars.exists()) {
hiveTestJars.mkdirs()
}
val targetFile = new File(hiveTestJars, filename)
if (!targetFile.exists() || !(targetFile.length() > 0)) {
val conf = new SparkConf
val securityManager = new org.apache.spark.SecurityManager(conf)
val hadoopConf = new Configuration
// propagate exceptions up to the caller of getFileFromUrl
Utils.doFetchFile(urlString, hiveTestJars, filename, conf, securityManager, hadoopConf)
}
targetFile
}There was a problem hiding this comment. Yes, that kind of thing. It could be a little simpler (e.g. no need for defs for the There was a problem hiding this comment. private def getJarFromUrl(urlString: String): File = {
val fileName = urlString.split("/").last
val targetFile = new File(hiveTestJarsDir, fileName)
if (!targetFile.exists()) {
Utils.doFetchFile(urlString, hiveTestJarsDir, fileName, new SparkConf, null, null)
}
targetFile
}The method I think it may be incorrect. Why not change to: private def getJarFromUrl(urlString: String): File = {
val fileName = urlString.split("/").last
Utils.doFetchFile(urlString, hiveTestJarsDir, fileName, new SparkConf, null, null)
}Because the There was a problem hiding this comment. Thank you @LantaoJin My change is to avoid downloading these two files many times. |
||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.