Harden Kafka integration tests for reliable nightly runs

docker/integration-tests/integration-tests-kafka.yaml

@@ -20,13 +20,21 @@ services:
     extends:
       file: integration-tests-base.yaml
       service: integration_test
+    environment:
+      # Available to hop-run / nested workflows as a variable (see run-tests.sh -p)
+      - BOOTSTRAP_SERVERS=kafka:9092
+    depends_on:
+      kafka:
+        condition: service_healthy
     links:
       - kafka
 
   kafka:
-    image: 'apache/kafka:latest'
-    ports:
-      - '9092'
+    # Pin to match lib kafka-clients.version; avoid :latest drift on nightly CI
+    image: 'apache/kafka:3.9.2'
+    hostname: kafka
+    # No host port publish: tests use the compose network (kafka:9092) only.
+    # Publishing 9092 on the host risks colliding with other services on ASF agents.
     environment:
       KAFKA_NODE_ID: 1
       KAFKA_PROCESS_ROLES: broker,controller
@@ -40,4 +48,14 @@ services:
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
       KAFKA_NUM_PARTITIONS: 3
-
+    healthcheck:
+      # Wait until the broker accepts connections on the PLAINTEXT listener
+      test:
+        [
+          "CMD-SHELL",
+          "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1 || exit 1",
+        ]
+      interval: 5s
+      timeout: 10s
+      retries: 20
+      start_period: 20s

integration-tests/kafka/main-0001-kafka-test-basic.hwf

@@ -30,8 +30,8 @@ limitations under the License.
   <parameters>
     <parameter>
       <name>BOOTSTRAP_SERVERS</name>
-      <description/>
-      <default_value/>
+      <description>Kafka bootstrap servers (docker IT network hostname)</description>
+      <default_value>kafka:9092</default_value>
     </parameter>
   </parameters>
   <actions>

integration-tests/kafka/main-0002-kafka-test-mapping.hwf

@@ -30,8 +30,8 @@ limitations under the License.
   <parameters>
     <parameter>
       <name>BOOTSTRAP_SERVERS</name>
-      <description/>
-      <default_value/>
+      <description>Kafka bootstrap servers (docker IT network hostname)</description>
+      <default_value>kafka:9092</default_value>
     </parameter>
   </parameters>
   <actions>

integration-tests/kafka/main-HOP3504-kafka-test-error-path.hwf

@@ -31,8 +31,8 @@ limitations under the License.
   <parameters>
     <parameter>
       <name>BOOTSTRAP_SERVERS</name>
-      <default_value/>
-      <description/>
+      <default_value>kafka:9092</default_value>
+      <description>Kafka bootstrap servers (docker IT network hostname)</description>
     </parameter>
   </parameters>
   <actions>

integration-tests/kafka/project-config.json

@@ -8,6 +8,10 @@
       "name" : "HOP_LICENSE_HEADER_FILE",
       "value" : "${PROJECT_HOME}/../asf-header.txt",
       "description" : "This will automatically serialize the ASF license header into pipelines and workflows in the integration test projects"
+    }, {
+      "name" : "BOOTSTRAP_SERVERS",
+      "value" : "kafka:9092",
+      "description" : "Kafka bootstrap servers for the docker integration-test network"
     } ]
   }
 }

integration-tests/scripts/run-tests.sh

@@ -71,6 +71,16 @@ chmod 777 "${CURRENT_DIR}"/../surefire-reports/ 2>/dev/null || true
 if [ -z "${BOOTSTRAP_SERVERS}" ]; then
   BOOTSTRAP_SERVERS=kafka:9092
 fi
+# Best-effort diagnostics when running the kafka project (helps Jenkins triage)
+if [ "${PROJECT_NAME}" = "kafka" ] || [ "$(basename "${PROJECT_NAME}" 2>/dev/null)" = "kafka" ]; then
+  echo "Kafka IT: BOOTSTRAP_SERVERS=${BOOTSTRAP_SERVERS}"
+  if command -v getent >/dev/null 2>&1; then
+    echo "Kafka IT: getent hosts kafka => $(getent hosts kafka 2>&1 || true)"
+  fi
+  if command -v nc >/dev/null 2>&1; then
+    nc -z -w 2 kafka 9092 && echo "Kafka IT: kafka:9092 is reachable" || echo "Kafka IT: kafka:9092 is NOT reachable yet"
+  fi
+fi
 
 # Get database parameters
 if [ -z "${POSTGRES_HOST}" ]; then

integration-tests/scripts/run-workflow.hwf

@@ -34,6 +34,11 @@ limitations under the License.
       <default_value/>
       <description>The filename of the workflow to execute</description>
     </parameter>
+    <parameter>
+      <name>BOOTSTRAP_SERVERS</name>
+      <default_value>kafka:9092</default_value>
+      <description>Kafka bootstrap servers (docker IT network hostname)</description>
+    </parameter>
   </parameters>
   <actions>
     <action>