Using apache kudu docker images and update kudu-client version (#626)

* update kudu connector to new version * revert not need changes * parse encryptionPolicy from conf file * fix ci fmt * revert a change
apache · Apr 23, 2024 · a8f1573 · a8f1573
1 parent 2848fdd
commit a8f1573
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 30 deletions.
diff --git a/.github/workflows/check-build-test.yml b/.github/workflows/check-build-test.yml
@@ -105,7 +105,7 @@ jobs:
           - { connector: jms,                          pre_cmd: 'docker-compose up -d ibmmq' }
           - { connector: json-streaming }
           - { connector: kinesis }
-          - { connector: kudu,                         pre_cmd: 'docker-compose up -d kudu-master-data kudu-tserver-data kudu-master kudu-tserver' }
+          - { connector: kudu,                         pre_cmd: 'docker-compose up -d kudu-master kudu-tserver' }
           - { connector: mongodb,                      pre_cmd: 'docker-compose up -d mongo' }
           - { connector: mqtt,                         pre_cmd: 'docker-compose up -d mqtt' }
           - { connector: mqtt-streaming,               pre_cmd: 'docker-compose up -d mqtt' }

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -178,32 +178,36 @@ services:
       - "AUTH_HOST=http://ironauth:8090"
     ports:
       - "8080:8080"
-  kudu-master-data:
-    image: kunickiaj/kudu
-    volumes:
-      - /var/lib/kudu/master
-  kudu-tserver-data:
-    image: kunickiaj/kudu
-    volumes:
-      - kudu-tserver-data:/var/lib/kudu/tserver
   kudu-master:
-    image: kunickiaj/kudu
+    image: apache/kudu:1.17.0
     ports:
-      - 7051:7051
-    volumes:
-      - kudu-tserver-data:/var/lib/kudu/tserver
-    command: master
-  kudu-tserver:
-    image: kunickiaj/kudu
+      - "7051:7051"
+    command: [ "master" ]
     environment:
-      - KUDU_MASTER=kudu-master
-    ports:
-      - 7050:7050
-    volumes:
-      - kudu-tserver-data:/var/lib/kudu/tserver
-    command: tserver
+      - KUDU_MASTERS=kudu-master:7051
+      - >
+        MASTER_ARGS=--fs_wal_dir=/var/lib/kudu/master
+        --rpc_bind_addresses=0.0.0.0:7051
+        --stderrthreshold=0
+        --use_hybrid_clock=false
+        --unlock_unsafe_flags=true
+  kudu-tserver:
+    image: apache/kudu:1.17.0
     links:
       - kudu-master
+    ports:
+      - "7050:7050"
+    command: [ "tserver" ]
+    environment:
+      - KUDU_MASTERS=kudu-master:7051
+      - >
+        TSERVER_ARGS=--fs_wal_dir=/var/lib/kudu/tserver
+        --rpc_bind_addresses=0.0.0.0:7050
+        --stderrthreshold=0
+        --use_hybrid_clock=false
+        --unlock_unsafe_flags=true
+    deploy:
+      replicas: 1
   mongo:
     image: mongo
     ports:

diff --git a/kudu/src/main/resources/reference.conf b/kudu/src/main/resources/reference.conf
@@ -4,5 +4,8 @@ pekko.connectors.kudu {
 
   # the hostname and port to locate the master
   master-address = ""
+  # options: OPTIONAL,REQUIRED_REMOTE,REQUIRED
+  # reference to org.apache.kudu.client.AsyncKuduClient.EncryptionPolicy
+  encryptionPolicy = "OPTIONAL"
 
 }
diff --git a/kudu/src/main/scala/org/apache/pekko/stream/connectors/kudu/KuduClientExt.scala b/kudu/src/main/scala/org/apache/pekko/stream/connectors/kudu/KuduClientExt.scala
@@ -16,16 +16,18 @@ package org.apache.pekko.stream.connectors.kudu
 import org.apache.pekko
 import pekko.actor.{ ClassicActorSystemProvider, ExtendedActorSystem, Extension, ExtensionId, ExtensionIdProvider }
 import org.apache.kudu.client.KuduClient
+import org.apache.kudu.client.AsyncKuduClient.EncryptionPolicy
 
 /**
  * Manages one [[org.apache.kudu.client.KuduClient]] per `ActorSystem`.
  */
 final class KuduClientExt private (sys: ExtendedActorSystem) extends Extension {
   val client = {
     val masterAddress = sys.settings.config.getString("pekko.connectors.kudu.master-address")
-    new KuduClient.KuduClientBuilder(masterAddress).build
+    new KuduClient.KuduClientBuilder(masterAddress)
+      .encryptionPolicy(EncryptionPolicy.valueOf(
+        sys.settings.config.getString("pekko.connectors.kudu.encryptionPolicy"))).build()
   }
-
   sys.registerOnTermination(client.shutdown())
 }
 

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -64,6 +64,7 @@ object Dependencies {
 
   val GoogleAuthVersion = "1.23.0"
   val JwtScalaVersion = "10.0.0"
+  val Log4jVersion = "2.23.1"
 
   // Releases https://github.com/FasterXML/jackson-databind/releases
   // CVE issues https://github.com/FasterXML/jackson-databind/issues?utf8=%E2%9C%93&q=+label%3ACVE
@@ -364,14 +365,11 @@ object Dependencies {
       "org.slf4j" % "slf4j-api" % Slf4jVersion % Test,
       "ch.qos.logback" % "logback-classic" % LogbackVersion % Test) ++ Mockito)
 
-  val KuduVersion = "1.10.1"
+  val KuduVersion = "1.17.0"
   val Kudu = Seq(
     libraryDependencies ++= Seq(
-      "org.apache.kudu" % "kudu-client-tools" % KuduVersion,
-      "org.apache.kudu" % "kudu-client" % KuduVersion % Test),
-    dependencyOverrides ++= Seq(
-      "org.slf4j" % "slf4j-api" % Slf4jLegacyVersion,
-      "ch.qos.logback" % "logback-classic" % LogbackLegacyVersion))
+      "org.apache.kudu" % "kudu-client" % KuduVersion,
+      "org.apache.logging.log4j" % "log4j-to-slf4j" % Log4jVersion % Test))
 
   val MongoDb = Seq(
     crossScalaVersions -= Scala3,