[SPARK-36734][SQL][BUILD][3.1] Upgrade ORC to 1.5.13

apache · Sep 16, 2021 · 72857b7 · 72857b7
1 parent 682442a
commit 72857b7
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 10 deletions.
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -195,9 +195,9 @@ objenesis/2.6//objenesis-2.6.jar
 okhttp/3.12.12//okhttp-3.12.12.jar
 okio/1.14.0//okio-1.14.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.12//orc-core-1.5.12.jar
-orc-mapreduce/1.5.12//orc-mapreduce-1.5.12.jar
-orc-shims/1.5.12//orc-shims-1.5.12.jar
+orc-core/1.5.13//orc-core-1.5.13.jar
+orc-mapreduce/1.5.13//orc-mapreduce-1.5.13.jar
+orc-shims/1.5.13//orc-shims-1.5.13.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar

diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -209,9 +209,9 @@ okhttp/2.7.5//okhttp-2.7.5.jar
 okhttp/3.12.12//okhttp-3.12.12.jar
 okio/1.14.0//okio-1.14.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.12//orc-core-1.5.12.jar
-orc-mapreduce/1.5.12//orc-mapreduce-1.5.12.jar
-orc-shims/1.5.12//orc-shims-1.5.12.jar
+orc-core/1.5.13//orc-core-1.5.13.jar
+orc-mapreduce/1.5.13//orc-mapreduce-1.5.13.jar
+orc-shims/1.5.13//orc-shims-1.5.13.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar

diff --git a/pom.xml b/pom.xml
@@ -136,7 +136,7 @@
     <kafka.version>2.6.0</kafka.version>
     <derby.version>10.12.1.1</derby.version>
     <parquet.version>1.10.1</parquet.version>
-    <orc.version>1.5.12</orc.version>
+    <orc.version>1.5.13</orc.version>
     <jetty.version>9.4.40.v20210413</jetty.version>
     <jakartaservlet.version>4.0.3</jakartaservlet.version>
     <chill.version>0.9.5</chill.version>
@@ -290,6 +290,11 @@
         <enabled>false</enabled>
       </snapshots>
     </repository>
+    <repository>
+      <id>apache.staging</id>
+      <name>Apache Staging Repository</name>
+      <url>https://repository.apache.org/content/repositories/orgapacheorc-1050</url>
+     </repository>
   </repositories>
   <pluginRepositories>
     <pluginRepository>

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
@@ -270,6 +270,7 @@ object SparkBuild extends PomBuild {
       "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/",
       DefaultMavenRepository,
       Resolver.mavenLocal,
+      "Apache Staging Repository" at "https://repository.apache.org/content/repositories/orgapacheorc-1050",
       Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
     ),
     externalResolvers := resolvers.value,

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
@@ -693,11 +693,11 @@ class FileBasedDataSourceSuite extends QueryTest
   }
 
   test("SPARK-22790,SPARK-27668: spark.sql.sources.compressionFactor takes effect") {
-    Seq(1.0, 0.5).foreach { compressionFactor =>
+    Seq(1.0, 0.4).foreach { compressionFactor =>
       withSQLConf(SQLConf.FILE_COMPRESSION_FACTOR.key -> compressionFactor.toString,
         SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "250") {
         withTempPath { workDir =>
-          // the file size is 486 bytes
+          // the file size is 504 bytes
           val workDirPath = workDir.getAbsolutePath
           val data1 = Seq(100, 200, 300, 400).toDF("count")
           data1.write.orc(workDirPath + "/data1")
@@ -706,7 +706,7 @@ class FileBasedDataSourceSuite extends QueryTest
           data2.write.orc(workDirPath + "/data2")
           val df2FromFile = spark.read.orc(workDirPath + "/data2")
           val joinedDF = df1FromFile.join(df2FromFile, Seq("count"))
-          if (compressionFactor == 0.5) {
+          if (compressionFactor == 0.4) {
             val bJoinExec = collect(joinedDF.queryExecution.executedPlan) {
               case bJoin: BroadcastHashJoinExec => bJoin
             }