Merge pull request #247 from dafreels/develop

KinesisPipelineDriver ConsumerStream Issue
Acxiom · Jun 28, 2021 · 9c1fdfc · 9c1fdfc
2 parents 1aa7b8b + 4aca888
commit 9c1fdfc
Show file tree

Hide file tree

Showing 13 changed files with 17 additions and 27 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -81,12 +81,6 @@ jobs:
         with:
           path: ~/.m2
           key: metalus-build-${{ env.cache-name }}
-      - id: get-versions
-        run: |
-          spark=$(echo ${{matrix.spark}} | cut -d_ -f1)
-          scala=$(echo ${{matrix.spark}} | cut -d_ -f2)
-          echo "::set-output name=spark::$spark"
-          echo "::set-output name=scala::$scala"
       - name: Remove SNAPSHOT
         run: mvn -B -P spark_${{ steps.get-versions.outputs.spark }},scala_${{ steps.get-versions.outputs.scala }} versions:set -DremoveSnapshot
       - name: Setup Signing Key
@@ -99,7 +93,7 @@ jobs:
         env:
           SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
-        run: mvn --settings deployment/release-settings.xml -pl . -B -P release,spark_${{ steps.get-versions.outputs.spark }},scala_${{ steps.get-versions.outputs.scala }} -DskipTests=true -Dgpg.executable=gpg1 -Dgpg.keyname=E08709FDFF083D6D -Dgpg.passphrase=${{ secrets.SIGNING_KEY_PASSPHRASE }} -Dgpg.publicKeyring=deployment/pubring.gpg -Dgpg.secretKeyring=deployment/secring.gpg deploy
+        run: mvn --settings deployment/release-settings.xml -pl . -B -P release,spark_2.4,scala_2.11 -DskipTests=true -Dgpg.executable=gpg1 -Dgpg.keyname=E08709FDFF083D6D -Dgpg.passphrase=${{ secrets.SIGNING_KEY_PASSPHRASE }} -Dgpg.publicKeyring=deployment/pubring.gpg -Dgpg.secretKeyring=deployment/secring.gpg deploy
   release:
     if: ${{ github.repository == 'Acxiom/metalus' && github.ref == 'refs/heads/master' && github.event_name != 'pull_request' }}
     name: Release

diff --git a/metalus-application/pom.xml b/metalus-application/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

diff --git a/metalus-aws/pom.xml b/metalus-aws/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-aws/src/main/scala/com/acxiom/aws/drivers/KinesisPipelineDriver.scala b/metalus-aws/src/main/scala/com/acxiom/aws/drivers/KinesisPipelineDriver.scala
@@ -9,7 +9,7 @@ import com.amazonaws.services.kinesis.model.Record
 import org.apache.log4j.Logger
 import org.apache.spark.rdd.RDD
 import org.apache.spark.storage.StorageLevel
-import org.apache.spark.streaming.kinesis.{KinesisInitialPositions, KinesisInputDStream, SparkAWSCredentials}
+import org.apache.spark.streaming.kinesis.{KinesisInitialPositions, KinesisInputDStream}
 import org.apache.spark.streaming.{Duration, StreamingContext, Time}
 
 import java.util.Date
@@ -62,7 +62,7 @@ object KinesisPipelineDriver {
     // Handle multiple shards
     val numShards = kinesisClient.describeStream(parameters("streamName").asInstanceOf[String]).getStreamDescription.getShards.size
     logger.info("Number of Kinesis shards is : " + numShards)
-    val numStreams = parameters.getOrElse("consumerStreams", numShards).asInstanceOf[String].toInt
+    val numStreams = parameters.getOrElse("consumerStreams", numShards).toString.toInt
     // Create the Kinesis DStreams
     val kinesisStreams = createKinesisDStreams(credentialProvider, appName, duration, streamingContext, numStreams, region, streamName)
     logger.info("Created " + kinesisStreams.size + " Kinesis DStreams")

diff --git a/metalus-common/pom.xml b/metalus-common/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
 
     <dependencies>

diff --git a/metalus-core/pom.xml b/metalus-core/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-delta/pom.xml b/metalus-delta/pom.xml
@@ -12,7 +12,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
 
     <dependencies>

diff --git a/metalus-examples/pom.xml b/metalus-examples/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-gcp/pom.xml b/metalus-gcp/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
 
     <dependencyManagement>
@@ -161,12 +161,8 @@
                         <configuration>
                             <relocations>
                                 <relocation>
-                                    <pattern>com.google.common</pattern>
-                                    <shadedPattern>metalus.com.google.common</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>com.google.protobuf</pattern>
-                                    <shadedPattern>metalus.com.google.protobuf</shadedPattern>
+                                    <pattern>com.google</pattern>
+                                    <shadedPattern>metalus.com.google</shadedPattern>
                                 </relocation>
                             </relocations>
                         </configuration>

diff --git a/metalus-kafka/pom.xml b/metalus-kafka/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/metalus-mongo/pom.xml b/metalus-mongo/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
 
     <dependencies>

diff --git a/metalus-utils/pom.xml b/metalus-utils/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>com.acxiom</groupId>
         <artifactId>metalus</artifactId>
-        <version>1.8.1-SNAPSHOT</version>
+        <version>1.8.2-SNAPSHOT</version>
     </parent>
 
     <properties>

diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.acxiom</groupId>
     <artifactId>metalus</artifactId>
-    <version>1.8.1-SNAPSHOT</version>
+    <version>1.8.2-SNAPSHOT</version>
     <name>${project.artifactId}</name>
     <packaging>pom</packaging>
     <description>Metalus Pipeline Library</description>
@@ -379,7 +379,7 @@
             <id>spark_3.1</id>
             <properties>
                 <spark.compat.version>3.1</spark.compat.version>
-                <spark.version>3.1.1</spark.version>
+                <spark.version>3.1.2</spark.version>
                 <json4s.version>3.6.6</json4s.version>
                 <mongo.version>3.0.0</mongo.version>
                 <scala.compat.version>2.12</scala.compat.version>