feature/strimzi-operator-connector

.github/workflows/main.yml

@@ -25,7 +25,7 @@ jobs:
           java-version: 17
 
       - name: Setup Gradle
-        uses: gradle/gradle-build-action@v2
+        uses: gradle/actions/setup-gradle@v3
 
       - name: Compile code
         run: ./gradlew assemble
@@ -65,14 +65,14 @@ jobs:
         uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Cache layers
         id: cache-buildx
         uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-s3-${{ hashFiles('Dockerfile.s3', 'src/**', '*.gradle') }}
+          key: ${{ runner.os }}-buildx-s3-${{ hashFiles('./docker/legacy/Dockerfile.s3', 'src/**', '*.gradle') }}
           restore-keys: |
             ${{ runner.os }}-buildx-s3-
             ${{ runner.os }}-buildx-
@@ -89,10 +89,10 @@ jobs:
           fi
 
       - name: Build backend docker
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
-          file: ./Dockerfile.s3
+          file: ././docker/legacy/Dockerfile.s3
           platforms: linux/amd64,linux/arm64
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: ${{ steps.cache-parameters.outputs.cache-to }}
@@ -104,10 +104,10 @@ jobs:
 
       # will use the internal cache from the previous build step, and load it into the current memory
       - name: Build backend docker locally
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: ./
-          file: ./Dockerfile.s3
+          file: ././docker/legacy/Dockerfile.s3
           cache-from: ${{ steps.cache-parameters.outputs.load-cache-from }}
           load: true
           tags: ${{ steps.docker_meta.outputs.tags }}
@@ -154,14 +154,14 @@ jobs:
         uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Cache frontend layers
         id: cache-buildx
         uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-generic-${{ hashFiles('Dockerfile', 'src/**', '*.gradle') }}
+          key: ${{ runner.os }}-buildx-generic-${{ hashFiles('./docker/legacy/Dockerfile', 'src/**', '*.gradle') }}
           restore-keys: |
             ${{ runner.os }}-buildx-generic-
             ${{ runner.os }}-buildx-
@@ -178,7 +178,7 @@ jobs:
           fi
 
       - name: Build docker
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64
@@ -191,7 +191,7 @@ jobs:
             ${{ steps.docker_meta.outputs.labels }}
 
       - name: Build docker locally
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           cache-from: ${{ steps.cache-parameters.outputs.load-cache-from }}

.github/workflows/release-dockerhub.yml

@@ -0,0 +1,92 @@
+# Create release files
+name: Release DockerHub
+
+on:
+  release:
+    types: [published]
+
+env:
+  DOCKER_IMAGE: radarbase/kafka-connect-transform-keyvalue
+  DOCKER_IMAGE_S3: radarbase/kafka-connect-transform-s3
+
+jobs:
+
+  # Build and push tagged release docker image
+  docker:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # Add Docker labels and tags
+      - name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_IMAGE }}
+          tags: |
+            type=match,pattern=v(.*),group=1
+
+      # Add Docker labels and tags
+      - name: Docker meta S3
+        id: docker_meta_s3
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_IMAGE_S3 }}
+          tags: |
+            type=match,pattern=v(.*),group=1
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: docker/legacy/Dockerfile
+          push: true
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          platforms: linux/arm64,linux/amd64
+          # Use runtime labels from docker_meta as well as fixed labels
+          labels: |
+            ${{ steps.docker_meta.outputs.labels }}
+            maintainer=Pim van Nierop<pim@thehyve.nl>
+            org.opencontainers.image.authors=Pim van Nierop<pim@thehyve.nl>, Pauline Conde <pauline.conde@kcl.ac.uk>
+            org.opencontainers.image.description=Key-value transformation for Kafka Connect
+            org.opencontainers.image.vendor=RADAR-base
+            org.opencontainers.image.licenses=Apache-2.0
+
+      - name: Build and push S3
+        id: docker_build_s3
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: docker/legacy/Dockerfile.s3
+          push: true
+          platforms: linux/arm64,linux/amd64
+          # Use runtime labels from docker_meta as well as fixed labels
+          labels: |
+            ${{ steps.docker_meta_s3.outputs.labels }}
+            maintainer=Pim van Nierop<pim@thehyve.nl>
+            org.opencontainers.image.authors=Pim van Nierop<pim@thehyve.nl>, Pauline Conde <pauline.conde@kcl.ac.uk>
+            org.opencontainers.image.description=Key-value transformation for Kafka Connect, with S3 connector loaded
+            org.opencontainers.image.vendor=RADAR-base
+            org.opencontainers.image.licenses=Apache-2.0
+
+      - name: Inspect image
+        run: |
+          docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+          docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+
+      - name: Inspect image S3
+        run: |
+          docker pull ${{ env.DOCKER_IMAGE_S3 }}:${{ steps.docker_meta_s3.outputs.version }}
+          docker image inspect ${{ env.DOCKER_IMAGE_S3 }}:${{ steps.docker_meta_s3.outputs.version }}

.github/workflows/release.yml

@@ -1,82 +1,76 @@
-# Create release files
+#See: https://docs.github.com/en/actions/publishing-packages/publishing-docker-images#publishing-images-to-github-packages
 name: Release
 
 on:
   release:
     types: [published]
 
 env:
-  DOCKER_IMAGE: radarbase/kafka-connect-transform-keyvalue
-  DOCKER_IMAGE_S3: radarbase/kafka-connect-transform-s3
+  REGISTRY: ghcr.io
+  DOCKER_IMAGE: radarbase/kafka-connect-transform-keyvalue-strimzi
+  DOCKER_IMAGE_S3: radarbase/kafka-connect-transform-s3-strimzi
 
 jobs:
+  # Upload jars to the GitHub release
   upload:
-    # The type of runner that the job will run on
     runs-on: ubuntu-latest
 
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v3
       - uses: actions/setup-java@v3
         with:
-          distribution: zulu
-          java-version: 11
+          distribution: temurin
+          java-version: 17
 
       - name: Setup Gradle
-        uses: gradle/gradle-build-action@v2
+        uses: gradle/actions/setup-gradle@v3
 
-      # Compile code
       - name: Compile code
         run: ./gradlew assemble
 
-      # Upload it to GitHub
       - name: Upload to GitHub
-        uses: AButler/upload-release-assets@v2.0.2
+        uses: AButler/upload-release-assets@v3.0
         with:
           files: 'build/libs/*;build/distributions/*'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
   # Build and push tagged release docker image
   docker:
-    # The type of runner that the job will run on
     runs-on: ubuntu-latest
 
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       - uses: actions/checkout@v3
 
-      # Add Docker labels and tags
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Docker meta
         id: docker_meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.DOCKER_IMAGE }}
+          images: ${{ env.REGISTRY }}/${{ env.DOCKER_IMAGE }}
           tags: |
             type=match,pattern=v(.*),group=1
 
-      # Add Docker labels and tags
       - name: Docker meta S3
         id: docker_meta_s3
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.DOCKER_IMAGE_S3 }}
+          images: ${{ env.REGISTRY }}/${{ env.DOCKER_IMAGE_S3 }}
           tags: |
             type=match,pattern=v(.*),group=1
 
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: true
@@ -85,36 +79,36 @@ jobs:
           # Use runtime labels from docker_meta as well as fixed labels
           labels: |
             ${{ steps.docker_meta.outputs.labels }}
-            maintainer=Joris Borgdorff <joris@thehyve.nl>
-            org.opencontainers.image.authors=Joris Borgdorff <joris@thehyve.nl>, Pauline Conde <pauline.conde@kcl.ac.uk>
-            org.opencontainers.image.description=Key-value transformation for Kafka Connect, with S3 connector loaded
+            maintainer=Pim van Nierop<pim@thehyve.nl>
+            org.opencontainers.image.authors=Pim van Nierop<pim@thehyve.nl>, Pauline Conde <pauline.conde@kcl.ac.uk>
+            org.opencontainers.image.description=Key-value transformation for Kafka Connect
             org.opencontainers.image.vendor=RADAR-base
             org.opencontainers.image.licenses=Apache-2.0
 
       - name: Build and push S3
         id: docker_build_s3
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: Dockerfile.s3
           push: true
           platforms: linux/arm64,linux/amd64
-          tags: ${{ steps.docker_meta_s3.outputs.tags }}
+
           # Use runtime labels from docker_meta as well as fixed labels
           labels: |
             ${{ steps.docker_meta_s3.outputs.labels }}
-            maintainer=Joris Borgdorff <joris@thehyve.nl>, Pauline Conde <pauline.conde@kcl.ac.uk>
-            org.opencontainers.image.authors=Joris Borgdorff <joris@thehyve.nl>, Pauline Conde <pauline.conde@kcl.ac.uk>
-            org.opencontainers.image.description=Key-value transformation for Kafka Connect
+            maintainer=Pim van Nierop<pim@thehyve.nl>
+            org.opencontainers.image.authors=Pim van Nierop<pim@thehyve.nl>, Pauline Conde <pauline.conde@kcl.ac.uk>
+            org.opencontainers.image.description=Key-value transformation for Kafka Connect, with S3 connector loaded
             org.opencontainers.image.vendor=RADAR-base
             org.opencontainers.image.licenses=Apache-2.0
 
       - name: Inspect image
         run: |
-          docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
-          docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+          docker pull ${{ env.REGISTRY }}/${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+          docker image inspect $${{ env.REGISTRY }}/{{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
 
       - name: Inspect image S3
         run: |
-          docker pull ${{ env.DOCKER_IMAGE_S3 }}:${{ steps.docker_meta_s3.outputs.version }}
-          docker image inspect ${{ env.DOCKER_IMAGE_S3 }}:${{ steps.docker_meta_s3.outputs.version }}
+          docker pull ${{ env.REGISTRY }}/${{ env.DOCKER_IMAGE_S3 }}:${{ steps.docker_meta_s3.outputs.version }}
+          docker image inspect ${{ env.REGISTRY }}/${{ env.DOCKER_IMAGE_S3 }}:${{ steps.docker_meta_s3.outputs.version }}

Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=confluentinc/cp-kafka-connect-base:7.8.1
+ARG BASE_IMAGE=quay.io/strimzi/kafka:0.46.0-kafka-3.9.0
 
 FROM --platform=$BUILDPLATFORM gradle:8.9-jdk17 AS builder
 
@@ -9,8 +9,12 @@ RUN gradle jar --no-watch-fs
 
 FROM ${BASE_IMAGE}
 
-ENV WAIT_FOR_KAFKA="1"
+ENV CONNECT_PLUGIN_PATH=/opt/kafka/plugins
 
-COPY --from=builder /code/build/libs/kafka-connect-transform-keyvalue*.jar /usr/share/"${COMPONENT}"/plugins/
-COPY ./src/main/docker/launch /etc/confluent/docker/launch
-COPY ./src/main/docker/kafka-wait /usr/bin/kafka-wait
+COPY --from=builder /code/build/libs/kafka-connect-transform-keyvalue*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-transform-keyvalue/
+
+USER 1001
+
+COPY --chown=1001:1001 ./docker/ensure /opt/kafka/ensure
+COPY --chown=1001:1001 ./docker/kafka_connect_run.sh /opt/kafka/kafka_connect_run.sh
+RUN chmod +x /opt/kafka/ensure /opt/kafka/kafka_connect_run.sh

Dockerfile.s3

@@ -1,4 +1,14 @@
-ARG BASE_IMAGE=confluentinc/cp-kafka-connect-base:7.8.1
+ARG BASE_IMAGE=quay.io/strimzi/kafka:0.46.0-kafka-3.9.0
+
+FROM confluentinc/cp-kafka-connect:8.0.0 AS hub
+
+ARG KAFKA_CONNECT_S3_VERSION=10.6.7
+# Version of Confluent components used by s3 connector plugin. Then upgrading the s3-connector plugin version, this version should be updated as well to the corresponding version.
+ARG CONFLUENT_VERSION=7.7.2
+
+RUN mkdir -p /tmp/deps/kafka-connect-s3/ /tmp/deps/kafka-connect-avro-converter/
+RUN confluent-hub install --no-prompt --component-dir /tmp/deps/kafka-connect-s3/ confluentinc/kafka-connect-s3:${KAFKA_CONNECT_S3_VERSION}
+RUN confluent-hub install --no-prompt --component-dir /tmp/deps/kafka-connect-avro-converter/ confluentinc/kafka-connect-avro-converter:${CONFLUENT_VERSION}
 
 FROM --platform=$BUILDPLATFORM gradle:8.9-jdk17 AS builder
 
@@ -9,13 +19,16 @@ RUN gradle jar --no-watch-fs
 
 FROM ${BASE_IMAGE}
 
-ENV WAIT_FOR_KAFKA="1"
-ARG KAFKA_CONNECT_S3_VERSION="10.5.23"
+USER root
 
-COPY --from=builder /code/build/libs/kafka-connect-transform-keyvalue*.jar /usr/share/"${COMPONENT}"/plugins/
-COPY ./src/main/docker/launch /etc/confluent/docker/launch
-COPY ./src/main/docker/kafka-wait /usr/bin/kafka-wait
+ENV CONNECT_PLUGIN_PATH=/opt/kafka/plugins
 
-USER root
-RUN confluent-hub install --no-prompt --component-dir /usr/share/"${COMPONENT}"/plugins confluentinc/kafka-connect-s3:"${KAFKA_CONNECT_S3_VERSION}"
-USER appuser
+COPY --from=builder /code/build/libs/kafka-connect-transform-keyvalue*.jar ${CONNECT_PLUGIN_PATH}/kafka-connect-transform-keyvalue/
+COPY --from=hub /tmp/deps/* ${CONNECT_PLUGIN_PATH}/
+RUN ln -s ${CONNECT_PLUGIN_PATH}/confluentinc-kafka-connect-avro-converter/lib/kafka-schema-registry-client*.jar ${CONNECT_PLUGIN_PATH}/confluentinc-kafka-connect-s3/lib/kafka-schema-registry-client.jar
+
+USER 1001
+
+COPY --chown=1001:1001 ./docker/ensure /opt/kafka/ensure
+COPY --chown=1001:1001 ./docker/kafka_connect_run.sh /opt/kafka/kafka_connect_run.sh
+RUN chmod +x /opt/kafka/ensure /opt/kafka/kafka_connect_run.sh