feat: merge from apache kafka 3.8 0971924ebc7e65eb7055010d2400626d31967d8c

.github/workflows/docker_build_and_test.yml

@@ -23,6 +23,7 @@ on:
         description: Docker image type to build and test
         options: 
           - "jvm"
+          - "native"
       kafka_url:
         description: Kafka url to be used to build the docker image
         required: true

.github/workflows/docker_official_image_build_and_test.yml

@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Docker Official Image Build Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      image_type:
+        type: choice
+        description: Docker image type to build and test
+        options: 
+          - "jvm"
+      kafka_version:
+        description: Kafka version for the docker official image. This should be >=3.7.0
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r docker/requirements.txt
+    - name: Build image and run tests
+      working-directory: ./docker
+      run: |
+        python docker_official_image_build_test.py kafka/test -tag=test -type=${{ github.event.inputs.image_type }} -v=${{ github.event.inputs.kafka_version }}
+    - name: Run CVE scan
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: 'kafka/test:test'
+        format: 'table'
+        severity: 'CRITICAL,HIGH'
+        output: scan_report_${{ github.event.inputs.image_type }}.txt
+        exit-code: '1'
+    - name: Upload test report
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: report_${{ github.event.inputs.image_type }}.html
+        path: docker/test/report_${{ github.event.inputs.image_type }}.html
+    - name: Upload CVE scan report
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: scan_report_${{ github.event.inputs.image_type }}.txt
+        path: scan_report_${{ github.event.inputs.image_type }}.txt

.github/workflows/docker_promote.yml

@@ -19,10 +19,10 @@ on:
   workflow_dispatch:
     inputs:
       rc_docker_image:
-        description: RC docker image that needs to be promoted (Example:- apache/kafka:3.6.0-rc0)
+        description: RC docker image that needs to be promoted (Example:- apache/kafka:3.8.0-rc0 (OR) apache/kafka-native:3.8.0-rc0)
         required: true
       promoted_docker_image:
-        description: Docker image name of the promoted image (Example:- apache/kafka:3.6.0)
+        description: Docker image name of the promoted image (Example:- apache/kafka:3.8.0 (OR) apache/kafka-native:3.8.0)
         required: true
 
 jobs:

.github/workflows/docker_rc_release.yml

@@ -23,8 +23,9 @@ on:
         description: Docker image type to be built and pushed
         options: 
           - "jvm"
+          - "native"
       rc_docker_image:
-        description: RC docker image that needs to be built and pushed to Dockerhub (Example:- apache/kafka:3.6.0-rc0)
+        description: RC docker image that needs to be built and pushed to Dockerhub (Example:- apache/kafka:3.8.0-rc0 (OR) apache/kafka-native:3.8.0-rc0)
         required: true
       kafka_url:
         description: Kafka url to be used to build the docker image

.github/workflows/docker_scan.yml

@@ -21,6 +21,7 @@ on:
   workflow_dispatch:
 jobs:
   scan_jvm:
+    if: github.repository == 'apache/kafka'
     runs-on: ubuntu-latest
     strategy:
       matrix:

.github/workflows/prepare_docker_official_image_source.yml

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Docker Prepare Docker Official Image Source
+
+on:
+  workflow_dispatch:
+    inputs:
+      image_type:
+        type: choice
+        description: Docker image type to build and test
+        options: 
+          - "jvm"
+      kafka_version:
+        description: Kafka version for the docker official image. This should be >=3.7.0
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r docker/requirements.txt
+    - name: Build Docker Official Image Artifact
+      working-directory: ./docker
+      run: |
+        python prepare_docker_official_image_source.py -type=${{ github.event.inputs.image_type }} -v=${{ github.event.inputs.kafka_version }}
+    - name: Upload Docker Official Image Artifact
+      if: success()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ github.event.inputs.kafka_version }}
+        path: docker/docker_official_images/${{ github.event.inputs.kafka_version }}

.gitignore

@@ -34,6 +34,8 @@ Vagrantfile.local
 
 config/server-*
 config/zookeeper-*
+gradle/wrapper/*.jar
+gradlew.bat
 
 results
 tests/results
@@ -58,4 +60,5 @@ jmh-benchmarks/src/main/generated
 storage/kafka-tiered-storage/
 
 docker/test/report_*.html
+kafka.Kafka
 __pycache__

Jenkinsfile

@@ -36,6 +36,12 @@ def doTest(env, target = "test") {
   junit '**/build/test-results/**/TEST-*.xml'
 }
 
+def runTestOnDevBranch(env) {
+  if (!isChangeRequest(env)) {
+    doTest(env)
+  }
+}
+
 def doStreamsArchetype() {
   echo 'Verify that Kafka Streams archetype compiles'
 
@@ -132,7 +138,7 @@ pipeline {
           }
           steps {
             doValidation()
-            doTest(env)
+            runTestOnDevBranch(env)
             echo 'Skipping Kafka Streams archetype test for Java 11'
           }
         }
@@ -151,7 +157,7 @@ pipeline {
           }
           steps {
             doValidation()
-            doTest(env)
+            runTestOnDevBranch(env)
             echo 'Skipping Kafka Streams archetype test for Java 17'
           }
         }

LICENSE-binary

@@ -60,12 +60,12 @@ reload4j-1.2.25
 rocksdbjni-7.9.2
 scala-collection-compat_2.12-2.10.0
 scala-collection-compat_2.13-2.10.0
-scala-library-2.12.18
-scala-library-2.13.12
+scala-library-2.12.19
+scala-library-2.13.14
 scala-logging_2.12-3.9.4
 scala-logging_2.13-3.9.4
-scala-reflect-2.12.18
-scala-reflect-2.13.12
+scala-reflect-2.12.19
+scala-reflect-2.13.14
 scala-java8-compat_2.12-1.0.2
 scala-java8-compat_2.13-1.0.2
 snappy-java-1.1.10.5

bin/kafka-run-class.sh

@@ -49,7 +49,7 @@ should_include_file() {
 base_dir=$(dirname $0)/..
 
 if [ -z "$SCALA_VERSION" ]; then
-  SCALA_VERSION=2.13.12
+  SCALA_VERSION=2.13.14
   if [[ -f "$base_dir/gradle.properties" ]]; then
     SCALA_VERSION=`grep "^scalaVersion=" "$base_dir/gradle.properties" | cut -d= -f 2`
   fi
@@ -208,7 +208,7 @@ fi
 
 # JMX settings
 if [ -z "$KAFKA_JMX_OPTS" ]; then
-  KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
+  KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
 fi
 
 # JMX port to use
@@ -353,9 +353,16 @@ CLASSPATH=${CLASSPATH#:}
 # If Cygwin is detected, classpath is converted to Windows format.
 (( WINDOWS_OS_FORMAT )) && CLASSPATH=$(cygpath --path --mixed "${CLASSPATH}")
 
-# Launch mode
-if [ "x$DAEMON_MODE" = "xtrue" ]; then
-  nohup "$JAVA" $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS $KAFKA_JMX_OPTS $KAFKA_LOG4J_CMD_OPTS $KAFKA_JDK_COMPATIBILITY_OPTS -cp "$CLASSPATH" $KAFKA_OPTS "$@" > "$CONSOLE_OUTPUT_FILE" 2>&1 < /dev/null &
+# If KAFKA_MODE=native, it will bring up Kafka in the native mode.
+# It expects the Kafka executable binary to be present at $base_dir/kafka.Kafka.
+# This is specifically used to run system tests on native Kafka - by bringing up Kafka in the native mode.
+if [[ "x$KAFKA_MODE" == "xnative" ]] && [[ "$*" == *"kafka.Kafka"* ]]; then
+  exec $base_dir/kafka.Kafka start --config "$2"  $KAFKA_LOG4J_CMD_OPTS $KAFKA_JMX_OPTS $KAFKA_OPTS
 else
-  exec "$JAVA" $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS $KAFKA_JMX_OPTS $KAFKA_LOG4J_CMD_OPTS $KAFKA_JDK_COMPATIBILITY_OPTS -cp "$CLASSPATH" $KAFKA_OPTS "$@"
+  # Launch mode
+  if [ "x$DAEMON_MODE" = "xtrue" ]; then
+    nohup "$JAVA" $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS $KAFKA_JMX_OPTS $KAFKA_LOG4J_CMD_OPTS $KAFKA_JDK_COMPATIBILITY_OPTS -cp "$CLASSPATH" $KAFKA_OPTS "$@" > "$CONSOLE_OUTPUT_FILE" 2>&1 < /dev/null &
+  else
+    exec "$JAVA" $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS $KAFKA_JMX_OPTS $KAFKA_LOG4J_CMD_OPTS $KAFKA_JDK_COMPATIBILITY_OPTS -cp "$CLASSPATH" $KAFKA_OPTS "$@"
+  fi
 fi

bin/windows/kafka-run-class.bat

@@ -27,7 +27,7 @@ set BASE_DIR=%CD%
 popd
 
 IF ["%SCALA_VERSION%"] EQU [""] (
-  set SCALA_VERSION=2.13.12
+  set SCALA_VERSION=2.13.14
 )
 
 IF ["%SCALA_BINARY_VERSION%"] EQU [""] (