[Spark Distribution] Default Assemble artifact to Spark 3 (#107)

* [Spark Distribution] Default to Spark 3 This changes the build.gradle file to default the assembled spark artifacts to use Spark 3 instead of Spark 2. This will enable users to leverage Spark 3 features (e.g., streaming, extensions). Signed-off-by: Nicholas Walter Knize <nknize@apache.org> * update changelog Signed-off-by: Nicholas Walter Knize <nknize@apache.org> --------- Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
opensearch-project · Feb 20, 2023 · 81fd694 · 81fd694
1 parent dd0396e
commit 81fd694
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Added
 - Added CHANGELOG and verifier workflow ([65](https://github.com/opensearch-project/opensearch-hadoop/pull/65))
 ### Changed
+- [Spark Distribution] Default Assemble artifact to Spark 3 ([107](https://github.com/opensearch-project/opensearch-hadoop/pull/107))
 ### Deprecated
 ### Removed
 ### Fixed

diff --git a/dist/build.gradle b/dist/build.gradle
@@ -8,7 +8,7 @@ apply plugin: 'opensearch.hadoop.build'
 description = "OpenSearch for Apache Hadoop"
 project.archivesBaseName = 'opensearch-hadoop'
 
-def sparkVariantIncluded = 'spark20scala211'
+def sparkVariantIncluded = 'spark30scala212'
 
 configurations {
     embedded {
@@ -44,11 +44,11 @@ configurations {
 BuildPlugin.disableTransitiveDependencies(project, project.configurations.thirdPartyShaded)
 
 def distProjects = [":opensearch-hadoop-mr", ":opensearch-hadoop-hive", ":opensearch-hadoop-pig",
-                    ":opensearch-spark-20", ":opensearch-storm"]
+                    ":opensearch-spark-30", ":opensearch-storm"]
 
 distProjects.each { distProject ->
     def configureDistDependency = { Dependency dependency ->
-        if (distProject == ":opensearch-spark-20") {
+        if (distProject == ":opensearch-spark-30") {
             dependency.capabilities {
                 requireCapability("org.opensearch.spark.sql.variant:$sparkVariantIncluded:$project.version")
             }

diff --git a/dist/licenses/commons-codec-1.15.jar.sha1 b/dist/licenses/commons-codec-1.15.jar.sha1
@@ -0,0 +1 @@
+49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
diff --git a/dist/licenses/commons-codec-1.4.jar.sha1 b/dist/licenses/commons-codec-1.4.jar.sha1