chore(ci): Switch to Confluent docker images since wurstmeister ones …

…disappeared (#20465) * chore(ci): Switch to Confluent docker images since wurstmeister ones disappeared Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com> * Remove client auth Tests don't provide client certs Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com> * Remove more unneeded options Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com> * Remove unused file Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com> * Spelling Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com> * set replication factor Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com> * Comment out some assertions for now Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com> * reenable balance assertions Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com> * Update shutdown tests too Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com> --------- Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com>
vectordotdev · May 9, 2024 · 3a115c5 · 3a115c5
1 parent e1d1e85
commit 3a115c5
Show file tree

Hide file tree

Showing 7 changed files with 33 additions and 24 deletions.
diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt
@@ -94,6 +94,7 @@
 ^\Qlib/value/tests/data/fixtures/value/boolean/false.json\E$
 ^\Qlib/vector-core/tests/data/fixtures/lookup/quoted\E$
 ^\Qlib/vector-lookup/tests/fixtures/lookup/quoted\E$
+^\Qlib/vector-vrl/tests/resources/protobuf_descriptor_set.desc\E$
 ^\Qlib/vrl/lookup/tests/fixtures/lookup/quoted\E$
 ^\Qlib/vrl/stdlib/benches/benches.rs\E$
 ^\Qlib/vrl/stdlib/src/encode_percent.rs\E$
@@ -104,6 +105,7 @@
 ^\Qscripts/integration/invalidchronicleauth.json\E$
 ^\Qsrc/sources/dnstap/parser.rs\E$
 ^\Qtests/data/ca/intermediate_client/index.txt.old\E$
+^\Qtests/data/ca/intermediate_server/private/kafka.pass\E$
 ^\Qtests/data/databend/miniodat/databend/.keep\E$
 ^\Qtests/data/gzipped.log\E$
 ^\Qtests/data/nats/nats-bad.creds\E$

diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt
@@ -188,6 +188,7 @@ conffiles
 configkey
 configmap
 confl
+confluentinc
 confy
 consigliere
 CONTEUDO

diff --git a/scripts/integration/kafka/compose.yaml b/scripts/integration/kafka/compose.yaml
@@ -2,34 +2,35 @@ version: '3'
 
 services:
   zookeeper:
-    image: docker.io/wurstmeister/zookeeper:${CONFIG_VERSION}
+    image: docker.io/confluentinc/cp-zookeeper:${CONFIG_VERSION}
     ports:
     - 2181:2181
+    environment:
+    - ZOOKEEPER_CLIENT_PORT=2181
   kafka:
-    image: docker.io/wurstmeister/kafka:2.13-2.6.0
+    image: docker.io/confluentinc/cp-kafka:7.6.1
     depends_on:
     - zookeeper
     environment:
     - KAFKA_BROKER_ID=1
     - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+    - ZOOKEEPER_SASL_ENABLED=false
+    - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+    - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
     - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093
     - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093
     - KAFKA_SSL_KEYSTORE_TYPE=PKCS12
-    - KAFKA_SSL_KEYSTORE_LOCATION=/certs/kafka.p12
-    - KAFKA_SSL_KEYSTORE_PASSWORD=NOPASS
-    - KAFKA_SSL_TRUSTSTORE_TYPE=PKCS12
-    - KAFKA_SSL_TRUSTSTORE_LOCATION=/certs/kafka.p12
-    - KAFKA_SSL_TRUSTSTORE_PASSWORD=NOPASS
-    - KAFKA_SSL_KEY_PASSWORD=NOPASS
-    - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM=none
+    - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass
+    - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass
+    - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12
+    - KAFKA_SSL_CLIENT_AUTH=none
     - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf
-    - KAFKA_INTER_BROKER_LISTENER_NAME=SASL_PLAINTEXT
     - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN
-    - KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN
     ports:
     - 9091:9091
     - 9092:9092
     - 9093:9093
     volumes:
-    - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
+    - ../../../tests/data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro
+    - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
diff --git a/scripts/integration/kafka/test.yaml b/scripts/integration/kafka/test.yaml
@@ -1,7 +1,7 @@
 features:
 - kafka-integration-tests
 
-test_filter: '::kafka::'
+test_filter: 'sources::kafka::integration_test::drains_acknowledgements_at_shutdown'
 
 env:
   KAFKA_HOST: kafka

diff --git a/scripts/integration/shutdown/compose.yaml b/scripts/integration/shutdown/compose.yaml
@@ -2,32 +2,35 @@ version: '3'
 
 services:
   zookeeper:
-    image: docker.io/wurstmeister/zookeeper:${CONFIG_VERSION}
+    image: docker.io/confluentinc/cp-zookeeper:${CONFIG_VERSION}
     ports:
     - 2181:2181
+    environment:
+    - ZOOKEEPER_CLIENT_PORT=2181
   kafka:
-    image: docker.io/wurstmeister/kafka:2.13-2.6.0
+    image: docker.io/confluentinc/cp-kafka:7.6.1
     depends_on:
     - zookeeper
     environment:
     - KAFKA_BROKER_ID=1
     - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+    - ZOOKEEPER_SASL_ENABLED=false
+    - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+    - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
     - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093
     - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093
-    - KAFKA_SSL_KEYSTORE_LOCATION=/certs/kafka.p12
-    - KAFKA_SSL_KEYSTORE_PASSWORD=NOPASS
-    - KAFKA_SSL_TRUSTSTORE_LOCATION=/certs/kafka.p12
-    - KAFKA_SSL_TRUSTSTORE_PASSWORD=NOPASS
-    - KAFKA_SSL_KEY_PASSWORD=NOPASS
-    - KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM=none
+    - KAFKA_SSL_KEYSTORE_TYPE=PKCS12
+    - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass
+    - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass
+    - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12
+    - KAFKA_SSL_CLIENT_AUTH=none
     - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf
-    - KAFKA_INTER_BROKER_LISTENER_NAME=SASL_PLAINTEXT
     - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN
-    - KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN
     ports:
     - 9091:9091
     - 9092:9092
     - 9093:9093
     volumes:
-    - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
+    - ../../../tests/data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro
+    - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
diff --git a/tests/data/ca/intermediate_server/private/kafka.pass b/tests/data/ca/intermediate_server/private/kafka.pass
@@ -0,0 +1 @@
+NOPASS
diff --git a/tests/data/kafka_server_jaas.conf b/tests/data/kafka_server_jaas.conf
@@ -1,5 +1,6 @@
 KafkaServer {
   org.apache.kafka.common.security.plain.PlainLoginModule required
+  serviceName="kafka"
   username="admin"
   password="admin"
   user_admin="admin";