Adding more workflows for each spark version (#202)

* Adding more workflows for each spark version Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com> * Adding additional workflows Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com> * Fix precommit Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com> --------- Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>
opensearch-project · May 1, 2023 · bbad040 · bbad040
1 parent a4aa70e
commit bbad040
Show file tree

Hide file tree

Showing 27 changed files with 206 additions and 18 deletions.
diff --git a/.github/workflows/build-spark-13.yml b/.github/workflows/build-spark-13.yml
@@ -0,0 +1,47 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build-and-test-spark-13:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest]
+    steps:
+      - name: Checkout Branch
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'gradle'
+
+      - name: 'Setup: Java 8 env'
+        run: echo "JAVA8_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+          cache: 'gradle'
+
+      - name: 'Setup: Java 11 env'
+        run: echo "JAVA11_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'gradle'
+
+      - name: 'Setup: Java 17 env'
+        run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Build with Gradle
+        run: ./gradlew opensearch-spark-13:build
diff --git a/.github/workflows/build-spark-20.yml b/.github/workflows/build-spark-20.yml
@@ -0,0 +1,47 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build-and-test-spark-20:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest]
+    steps:
+      - name: Checkout Branch
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'gradle'
+
+      - name: 'Setup: Java 8 env'
+        run: echo "JAVA8_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+          cache: 'gradle'
+
+      - name: 'Setup: Java 11 env'
+        run: echo "JAVA11_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'gradle'
+
+      - name: 'Setup: Java 17 env'
+        run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Build with Gradle
+        run: ./gradlew opensearch-spark-20:build
diff --git a/.github/workflows/build-spark-30.yml b/.github/workflows/build-spark-30.yml
@@ -0,0 +1,47 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build-and-test-spark-30:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest]
+    steps:
+      - name: Checkout Branch
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'gradle'
+
+      - name: 'Setup: Java 8 env'
+        run: echo "JAVA8_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+          cache: 'gradle'
+
+      - name: 'Setup: Java 11 env'
+        run: echo "JAVA11_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'gradle'
+
+      - name: 'Setup: Java 17 env'
+        run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      - name: Build with Gradle
+        run: ./gradlew opensearch-spark-30:build
diff --git a/gradle.properties b/gradle.properties
@@ -30,7 +30,7 @@ spark13Version = 1.6.2
 spark20Version = 2.3.0
 spark22Version = 2.2.3
 spark24Version = 2.4.4
-spark30Version = 3.2.3
+spark30Version = 3.2.4
 
 # same as Spark's
 scala210Version = 2.10.7

diff --git a/licenses/spark-catalyst_2.12-3.2.3.jar.sha1 b/licenses/spark-catalyst_2.12-3.2.3.jar.sha1
diff --git a/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 b/licenses/spark-catalyst_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+bfc16bcbcc2d6e65c61b1f3214c56e7ea998de90
diff --git a/licenses/spark-core_2.12-3.2.3.jar.sha1 b/licenses/spark-core_2.12-3.2.3.jar.sha1
diff --git a/licenses/spark-core_2.12-3.2.4.jar.sha1 b/licenses/spark-core_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+c61b811e273c04bb53931ab02d2648248033c445
diff --git a/licenses/spark-sql_2.12-3.2.3.jar.sha1 b/licenses/spark-sql_2.12-3.2.3.jar.sha1
diff --git a/licenses/spark-sql_2.12-3.2.4.jar.sha1 b/licenses/spark-sql_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+9bbe7a5fd0478bb7f3a2eb2e5dcb004f4ed70305
diff --git a/licenses/spark-streaming_2.12-3.2.3.jar.sha1 b/licenses/spark-streaming_2.12-3.2.3.jar.sha1
diff --git a/licenses/spark-streaming_2.12-3.2.4.jar.sha1 b/licenses/spark-streaming_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+ce150a20b26405f5b209205a4b64bc570d5a551f
diff --git a/licenses/spark-yarn_2.12-3.2.3.jar.sha1 b/licenses/spark-yarn_2.12-3.2.3.jar.sha1
diff --git a/licenses/spark-yarn_2.12-3.2.4.jar.sha1 b/licenses/spark-yarn_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+48f903127f7c2b69daaf30200d19d4d750e5af4a
diff --git a/...sql-20/src/itest/scala/org/opensearch/spark/sql/streaming/StreamingQueryTestHarness.scala b/...sql-20/src/itest/scala/org/opensearch/spark/sql/streaming/StreamingQueryTestHarness.scala
@@ -41,7 +41,7 @@ import org.apache.spark.sql.streaming.StreamingQuery
 import org.apache.spark.sql.streaming.StreamingQueryException
 import org.apache.spark.sql.streaming.StreamingQueryListener
 import org.junit.Assert
-import org.opensearch.hadoop.util.IOUtils
+import org.opensearch.hadoop.util.TestUtils
 import org.opensearch.hadoop.util.unit.TimeValue
 
 /**
@@ -396,11 +396,11 @@ class StreamingQueryTestHarness[S <: java.io.Serializable : Encoder](val sparkSe
 
 object TestingSerde extends Serializable {
   def serialize(any: java.io.Serializable): String = {
-    IOUtils.serializeToBase64(any)
+    TestUtils.serializeToBase64(any)
   }
 
   def deserialize[T](line: String): T = {
-    val data: T = IOUtils.deserializeFromBase64(line, T.class)
+    val data: T = TestUtils.deserializeFromBase64(line)
     data
   }
 }
diff --git a/spark/sql-30/licenses/spark-catalyst_2.12-3.2.3.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.12-3.2.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+bfc16bcbcc2d6e65c61b1f3214c56e7ea998de90
diff --git a/spark/sql-30/licenses/spark-core_2.12-3.2.3.jar.sha1 b/spark/sql-30/licenses/spark-core_2.12-3.2.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-core_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-core_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+c61b811e273c04bb53931ab02d2648248033c445
diff --git a/spark/sql-30/licenses/spark-sql_2.12-3.2.3.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.12-3.2.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-sql_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+9bbe7a5fd0478bb7f3a2eb2e5dcb004f4ed70305
diff --git a/spark/sql-30/licenses/spark-streaming_2.12-3.2.3.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.12-3.2.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-streaming_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+ce150a20b26405f5b209205a4b64bc570d5a551f
diff --git a/spark/sql-30/licenses/spark-yarn_2.12-3.2.3.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.12-3.2.3.jar.sha1
diff --git a/spark/sql-30/licenses/spark-yarn_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.12-3.2.4.jar.sha1
@@ -0,0 +1 @@
+48f903127f7c2b69daaf30200d19d4d750e5af4a
diff --git a/...sql-30/src/itest/scala/org/opensearch/spark/sql/streaming/StreamingQueryTestHarness.scala b/...sql-30/src/itest/scala/org/opensearch/spark/sql/streaming/StreamingQueryTestHarness.scala
@@ -41,7 +41,7 @@ import org.apache.spark.sql.streaming.StreamingQuery
 import org.apache.spark.sql.streaming.StreamingQueryException
 import org.apache.spark.sql.streaming.StreamingQueryListener
 import org.junit.Assert
-import org.opensearch.hadoop.util.IOUtils
+import org.opensearch.hadoop.util.TestUtils
 import org.opensearch.hadoop.util.unit.TimeValue
 
 /**
@@ -396,11 +396,11 @@ class StreamingQueryTestHarness[S <: java.io.Serializable : Encoder](val sparkSe
 
 object TestingSerde extends Serializable {
   def serialize(any: java.io.Serializable): String = {
-    IOUtils.serializeToBase64(any)
+    TestUtils.serializeToBase64(any)
   }
 
   def deserialize[T](line: String): T = {
-    val data: T = IOUtils.deserializeFromBase64(line, T.class)
+    val data: T = TestUtils.deserializeFromBase64(line, T.class)
     data
   }
 }
diff --git a/test/shared/src/main/java/org/opensearch/hadoop/util/TestUtils.java b/test/shared/src/main/java/org/opensearch/hadoop/util/TestUtils.java
@@ -7,7 +7,7 @@
  * Modifications Copyright OpenSearch Contributors. See
  * GitHub history for details.
  */
- 
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -28,13 +28,22 @@
  */
 package org.opensearch.hadoop.util;
 
+import static org.opensearch.hadoop.util.IOUtils.close;
+
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
 import java.util.Locale;
 
+import javax.xml.bind.DatatypeConverter;
+
+import org.opensearch.hadoop.OpenSearchHadoopIllegalStateException;
 import org.opensearch.hadoop.rest.RestClient;
+import org.opensearch.hadoop.serialization.OpenSearchHadoopSerializationException;
 
 public class TestUtils {
 
@@ -104,4 +113,42 @@ public static byte[] fromInputStream(InputStream in) throws IOException {
 
         return out.toByteArray();
     }
+
+    public static String serializeToBase64(Serializable object) {
+        if (object == null) {
+            return StringUtils.EMPTY;
+        }
+        FastByteArrayOutputStream baos = new FastByteArrayOutputStream();
+        ObjectOutputStream oos = null;
+        try {
+            oos = new ObjectOutputStream(baos);
+            oos.writeObject(object);
+        } catch (IOException ex) {
+            throw new OpenSearchHadoopSerializationException("Cannot serialize object " + object, ex);
+        } finally {
+            close(oos);
+        }
+        return DatatypeConverter.printBase64Binary(baos.bytes().bytes());
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <T extends Serializable> T deserializeFromBase64(String data) {
+        if (!StringUtils.hasLength(data)) {
+            return null;
+        }
+
+        byte[] rawData = DatatypeConverter.parseBase64Binary(data);
+        ObjectInputStream ois = null;
+        try {
+            ois = new ObjectInputStream(new FastByteArrayInputStream(rawData));
+            Object o = ois.readObject();
+            return (T) o;
+        } catch (ClassNotFoundException ex) {
+            throw new OpenSearchHadoopIllegalStateException("cannot deserialize object", ex);
+        } catch (IOException ex) {
+            throw new OpenSearchHadoopSerializationException("cannot deserialize object", ex);
+        } finally {
+            close(ois);
+        }
+    }
 }