From 2fc432209a1ae48e7b57fe7bab57256c091c9ff0 Mon Sep 17 00:00:00 2001 From: Pim van Nierop Date: Mon, 1 Sep 2025 10:00:39 +0200 Subject: [PATCH 1/6] Make release action more efficient and inline with other RADAR-base repos --- .github/workflows/release.yml | 127 ++++++++++++++++------------------ 1 file changed, 60 insertions(+), 67 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a978951..9bf5a8b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,3 @@ -# Create release files name: Release on: @@ -6,118 +5,112 @@ on: types: [published] env: - DOCKER_IMAGE: ghcr.io/${{ github.repository }}/kafka-connect-transform-keyvalue - DOCKER_IMAGE_S3: ghcr.io/${{ github.repository }}/kafka-connect-transform-s3 + REGISTRY: ghcr.io + REPOSITORY: ${{ github.repository }} + IMAGES: >- + [{ + 'name': 'kafka-connect-transform-keyvalue', + 'build_file': 'Dockerfile', + 'authors': 'Pim van Nierop ', + 'description': 'Key-value transformation for Kafka Connect' + },{ + 'name': 'kafka-connect-transform-s3', + 'build_file': 'Dockerfile.s3', + 'authors': 'Pim van Nierop ', + 'description': 'Key-value transformation for Kafka Connect, with S3 connector loaded' + }] jobs: upload: - # The type of runner that the job will run on runs-on: ubuntu-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: actions/setup-java@v3 with: - distribution: zulu + distribution: temurin java-version: 17 - name: Setup Gradle uses: gradle/gradle-build-action@v3 - # Compile code - name: Compile code run: ./gradlew assemble - # Upload it to GitHub - name: Upload to GitHub uses: AButler/upload-release-assets@v2.0.2 with: files: 'build/libs/*;build/distributions/*' repo-token: ${{ secrets.GITHUB_TOKEN }} - # Build and push tagged release docker image + prepare-matrix: + name: Prepare Matrix Output + runs-on: ubuntu-latest + outputs: + images: ${{ steps.step1.outputs.matrix }} + steps: + - name: Create Matrix Variable + id: step1 + run: echo "matrix=${{ env.IMAGES }}" >> $GITHUB_OUTPUT + docker: - # The type of runner that the job will run on + needs: prepare-matrix runs-on: ubuntu-latest + strategy: + matrix: + image: ${{ fromJson(needs.prepare-matrix.outputs.images ) }} + permissions: + contents: read + packages: write - # Steps represent a sequence of tasks that will be executed as part of the job steps: - uses: actions/checkout@v3 - # Add Docker labels and tags - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.DOCKER_IMAGE }} - tags: | - type=match,pattern=v(.*),group=1 + # Setup docker build environment + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - # Add Docker labels and tags - - name: Docker meta S3 - id: docker_meta_s3 - uses: docker/metadata-action@v4 - with: - images: ${{ env.DOCKER_IMAGE_S3 }} - tags: | - type=match,pattern=v(.*),group=1 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 - name: Login to Container Registry uses: docker/login-action@v2 with: - registry: ghcr.io + registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 + - name: Lowercase image name + run: | + echo "DOCKER_IMAGE=${REGISTRY}/${REPOSITORY,,}/${{ matrix.image.name }}" >>${GITHUB_ENV} - - name: Build and push - id: docker_build - uses: docker/build-push-action@v3 + # Add Docker labels and tags + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v4 with: - context: . - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - platforms: linux/arm64,linux/amd64 - labels: | - ${{ steps.docker_meta.outputs.labels }} - maintainer=Pim van Nierop - org.opencontainers.image.authors=Pim van Nierop - org.opencontainers.image.vendor=RADAR-base - org.opencontainers.image.licenses=Apache-2.0 - org.opencontainers.image.description=Key-value transformation for Kafka Connect + images: ${{ env.DOCKER_IMAGE }} + tags: | + type=match,pattern=v(.*),group=1 - - name: Build and push S3 - id: docker_build_s3 + - name: Build docker and push uses: docker/build-push-action@v3 with: context: . - file: Dockerfile.s3 + file: ${{ matrix.image.build_file }} + platforms: linux/amd64,linux/arm64 push: true - platforms: linux/arm64,linux/amd64 - tags: ${{ steps.docker_meta_s3.outputs.tags }} + tags: ${{ steps.docker_meta.outputs.tags }} labels: | - ${{ steps.docker_meta_s3.outputs.labels }} - maintainer=Pim van Nierop - org.opencontainers.image.authors=Pim van Nierop + ${{ steps.docker_meta.outputs.labels }} + maintainer=${{ matrix.image.authors }} + org.opencontainers.image.description=${{ matrix.image.description }} + org.opencontainers.image.authors=${{ matrix.image.authors }} org.opencontainers.image.vendor=RADAR-base org.opencontainers.image.licenses=Apache-2.0 - org.opencontainers.image.description=Key-value transformation for Kafka Connect, with S3 connector loaded - - - name: Lowercase REPO - run: | - echo "REPO=${GITHUB_REPOSITORY,,}" >>${GITHUB_ENV} - - - name: Inspect image - run: | - docker pull ghcr.io/${{ env.REPO }}/kafka-connect-transform-keyvalue:${{ steps.docker_meta.outputs.version }} - docker image inspect ghcr.io/${{ env.REPO }}/kafka-connect-transform-keyvalue:${{ steps.docker_meta.outputs.version }} - - name: Inspect image S3 + - name: Inspect docker image run: | - docker pull ghcr.io/${{ env.REPO }}/kafka-connect-transform-s3:${{ steps.docker_meta_s3.outputs.version }} - docker image inspect ghcr.io/${{ env.REPO }}/kafka-connect-transform-s3:${{ steps.docker_meta_s3.outputs.version }} + docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} + docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} From eafe29d021db151618a83fbbe2879a6332089c46 Mon Sep 17 00:00:00 2001 From: Pim van Nierop Date: Mon, 1 Sep 2025 10:21:45 +0200 Subject: [PATCH 2/6] Fix deprecated Snyk scan action --- .github/workflows/snyk.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snyk.yaml b/.github/workflows/snyk.yaml index 50c1596..32348e7 100644 --- a/.github/workflows/snyk.yaml +++ b/.github/workflows/snyk.yaml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v3 - name: Run Snyk to check for vulnerabilities - uses: snyk/actions/gradle-jdk17@master + uses: snyk/actions/gradle-8-jdk17@master env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} with: From 0f26fa496cbc9e170c25201a4221046937ab5694 Mon Sep 17 00:00:00 2001 From: Pim van Nierop Date: Mon, 1 Sep 2025 10:23:32 +0200 Subject: [PATCH 3/6] Update scheduled docker image scan action --- .github/workflows/scheduled-snyk-docker.yaml | 75 +++++++++++--------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/.github/workflows/scheduled-snyk-docker.yaml b/.github/workflows/scheduled-snyk-docker.yaml index f71de6c..e07ada9 100644 --- a/.github/workflows/scheduled-snyk-docker.yaml +++ b/.github/workflows/scheduled-snyk-docker.yaml @@ -6,64 +6,71 @@ on: workflow_dispatch: env: - DOCKER_IMAGE: radarbase/kafka-connect-transform-keyvalue - DOCKER_IMAGE_S3: radarbase/kafka-connect-transform-s3 + REGISTRY: ghcr.io + REPOSITORY: ${{ github.repository }} + IMAGES: >- + [{ + 'name': 'kafka-connect-transform-keyvalue', + 'build_file': 'Dockerfile', + 'authors': 'Pim van Nierop ', + 'description': 'Key-value transformation for Kafka Connect' + },{ + 'name': 'kafka-connect-transform-s3', + 'build_file': 'Dockerfile.s3', + 'authors': 'Pim van Nierop ', + 'description': 'Key-value transformation for Kafka Connect, with S3 connector loaded' + }] jobs: + prepare-matrix: + name: Prepare Matrix Output + runs-on: ubuntu-latest + outputs: + images: ${{ steps.step1.outputs.matrix }} + steps: + - name: Create Matrix Variable + id: step1 + run: | + echo "${{ env.IMAGES }}" + echo "matrix=${{ env.IMAGES }}" >> $GITHUB_OUTPUT + security: + needs: prepare-matrix runs-on: ubuntu-latest + strategy: + matrix: + image: ${{ fromJson(needs.prepare-matrix.outputs.images ) }} + permissions: + contents: read steps: - uses: actions/checkout@v3 - - name: Run Snyk to check for vulnerabilities in keyvalue image - continue-on-error: true # To make sure that SARIF upload gets called - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: ${{ env.DOCKER_IMAGE }} - # 'exclude-app-vulns' only tests vulnerabilities in the base image. - # Code base vulnerabilities are tested the scheduled-snyk.yaml action. - args: >- - --file=Dockerfile - --fail-on=upgradable - --severity-threshold=high - --policy-path=.snyk - --exclude-app-vulns - --org=radar-base - --sarif-file-output=snyk.sarif - - # Detected vulnerabilities will appear on Github in Security->Code_scanning_alerts tab - - name: Upload result to GitHub Code Scanning - uses: github/codeql-action/upload-sarif@v3 - with: - category: keyvalue - sarif_file: snyk.sarif + - name: Lowercase image name + run: | + echo "DOCKER_IMAGE=${REGISTRY}/${REPOSITORY,,}/${{ matrix.image.name }}" >>${GITHUB_ENV} - - name: Run Snyk to check for vulnerabilities in s3 connector image + - name: Run Snyk to check for vulnerabilities on docker image continue-on-error: true # To make sure that SARIF upload gets called uses: snyk/actions/docker@master env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} with: - image: ${{ env.DOCKER_IMAGE_S3 }} + image: ${{ env.DOCKER_IMAGE }} # 'exclude-app-vulns' only tests vulnerabilities in the base image. # Code base vulnerabilities are tested the scheduled-snyk.yaml action. args: >- - --file=Dockerfile + --file=${{ matrix.image.build_file }} --fail-on=upgradable --severity-threshold=high --policy-path=.snyk --exclude-app-vulns --org=radar-base - --sarif-file-output=snyk.sarif + --sarif-file-output=${{ matrix.image.name }}.sarif # Detected vulnerabilities will appear on Github in Security->Code_scanning_alerts tab - name: Upload result to GitHub Code Scanning uses: github/codeql-action/upload-sarif@v3 with: - category: s3 - sarif_file: snyk.sarif - - + sarif_file: ${{ matrix.image.name }}.sarif + category: ${{ matrix.image.name }} From 40c85afb00a9b9213f2f6e20b6226f1cd3c837f4 Mon Sep 17 00:00:00 2001 From: Pim van Nierop Date: Mon, 1 Sep 2025 10:24:22 +0200 Subject: [PATCH 4/6] Up project version to 7.8.4 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 611e87f..8ec64d4 100644 --- a/build.gradle +++ b/build.gradle @@ -3,7 +3,7 @@ plugins { id 'java-library' } -version = '7.8.3' +version = '7.8.4' description = "Kafka Connect transformation used to copy the key and value to a struct in the value of the record." sourceCompatibility = JavaVersion.VERSION_17 From c1495852cbd39c24053868a277aab2cc4ee4ce70 Mon Sep 17 00:00:00 2001 From: Pim van Nierop Date: Mon, 1 Sep 2025 12:19:03 +0200 Subject: [PATCH 5/6] Push docker images from main.yaml to GitHub Container Registry --- .github/workflows/main.yml | 157 ++++++++++++------------------------- 1 file changed, 50 insertions(+), 107 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 09cd85e..1529bee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,15 +8,27 @@ on: pull_request: branches: [ master, dev ] +env: + REGISTRY: ghcr.io + REPOSITORY: ${{ github.repository }} + IMAGES: >- + [{ + 'name': 'kafka-connect-transform-keyvalue', + 'build_file': 'Dockerfile', + 'authors': 'Pim van Nierop ', + 'description': 'Key-value transformation for Kafka Connect' + },{ + 'name': 'kafka-connect-transform-s3', + 'build_file': 'Dockerfile.s3', + 'authors': 'Pim van Nierop ', + 'description': 'Key-value transformation for Kafka Connect, with S3 connector loaded' + }] + jobs: - # Build and test the code kotlin: - # The type of runner that the job will run on runs-on: ubuntu-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: actions/setup-java@v3 @@ -34,37 +46,36 @@ jobs: - name: Check run: ./gradlew check - # Build and push tagged release backend docker image - dockerS3: - # The type of runner that the job will run on + prepare-matrix: + name: Prepare Matrix Output runs-on: ubuntu-latest + outputs: + images: ${{ steps.step1.outputs.matrix }} + steps: + - name: Create Matrix Variable + id: step1 + run: echo "matrix=${{ env.IMAGES }}" >> $GITHUB_OUTPUT - env: - DOCKER_IMAGE: radarbase/kafka-connect-transform-s3 + # Build and push tagged release backend docker image + docker: + needs: prepare-matrix + runs-on: ubuntu-latest + strategy: + matrix: + image: ${{ fromJson(needs.prepare-matrix.outputs.images ) }} + permissions: + contents: read + packages: write - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # Add Docker labels and tags - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.DOCKER_IMAGE }} - # Setup docker build environment - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx + id: buildx uses: docker/setup-buildx-action@v2 - name: Cache layers @@ -72,7 +83,7 @@ jobs: uses: actions/cache@v3 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-s3-${{ hashFiles('Dockerfile.s3', 'src/**', '*.gradle') }} + key: ${{ runner.os }}-buildx-s3-${{ hashFiles(matrix.image.build_file, 'src/**', '*.gradle') }} restore-keys: | ${{ runner.os }}-buildx-s3- ${{ runner.os }}-buildx- @@ -88,59 +99,17 @@ jobs: echo "load-cache-from=type=local,src=/tmp/.buildx-cache-new" >> $GITHUB_OUTPUT fi - - name: Build backend docker - uses: docker/build-push-action@v3 - with: - context: . - file: ./Dockerfile.s3 - platforms: linux/amd64,linux/arm64 - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: ${{ steps.cache-parameters.outputs.cache-to }} - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.docker_meta.outputs.tags }} - # Use runtime labels from docker_meta_backend as well as fixed labels - labels: | - ${{ steps.docker_meta.outputs.labels }} - - # will use the internal cache from the previous build step, and load it into the current memory - - name: Build backend docker locally - uses: docker/build-push-action@v3 + - name: Login to Container Registry + uses: docker/login-action@v2 with: - context: ./ - file: ./Dockerfile.s3 - cache-from: ${{ steps.cache-parameters.outputs.load-cache-from }} - load: true - tags: ${{ steps.docker_meta.outputs.tags }} + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Inspect docker image - run: docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move docker build cache - if: steps.cache-buildx.outputs.cache-hit != 'true' + - name: Lowercase image name run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - docker: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - env: - DOCKER_IMAGE: radarbase/kafka-connect-transform-keyvalue - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + echo "DOCKER_IMAGE=${REGISTRY}/${REPOSITORY,,}/${{ matrix.image.name }}" + echo "DOCKER_IMAGE=${REGISTRY}/${REPOSITORY,,}/${{ matrix.image.name }}" >>${GITHUB_ENV} # Add Docker labels and tags - name: Docker meta @@ -149,57 +118,31 @@ jobs: with: images: ${{ env.DOCKER_IMAGE }} - # Setup docker build environment - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache frontend layers - id: cache-buildx - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-generic-${{ hashFiles('Dockerfile', 'src/**', '*.gradle') }} - restore-keys: | - ${{ runner.os }}-buildx-generic- - ${{ runner.os }}-buildx- - - - name: Cache parameters - id: cache-parameters - run: | - if [ "${{ steps.cache-buildx.outputs.cache-hit }}" = "true" ]; then - echo "cache-to=" >> $GITHUB_OUTPUT - echo "load-cache-from=type=local,src=/tmp/.buildx-cache" >> $GITHUB_OUTPUT - else - echo "cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max" >> $GITHUB_OUTPUT - echo "load-cache-from=type=local,src=/tmp/.buildx-cache-new" >> $GITHUB_OUTPUT - fi - - - name: Build docker + - name: Build docker and optionally push uses: docker/build-push-action@v3 with: context: . + file: ${{ matrix.image.build_file }} platforms: linux/amd64,linux/arm64 cache-from: type=local,src=/tmp/.buildx-cache cache-to: ${{ steps.cache-parameters.outputs.cache-to }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_meta.outputs.tags }} - # Use runtime labels from docker_meta as well as fixed labels labels: | ${{ steps.docker_meta.outputs.labels }} - - name: Build docker locally + - name: Build docker using cache and load uses: docker/build-push-action@v3 with: context: . + file: ${{ matrix.image.build_file }} cache-from: ${{ steps.cache-parameters.outputs.load-cache-from }} load: true tags: ${{ steps.docker_meta.outputs.tags }} - name: Inspect docker image - run: docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} + run: | + docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} # Temp fix # https://github.com/docker/build-push-action/issues/252 From 6f899f70cf4e472f50cab245c890eb64afaf9d3d Mon Sep 17 00:00:00 2001 From: Pim van Nierop Date: Mon, 1 Sep 2025 12:54:37 +0200 Subject: [PATCH 6/6] Potential fix for code scanning alert no. 41: Workflow does not contain permissions Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1529bee..518f5f9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,6 +48,7 @@ jobs: prepare-matrix: name: Prepare Matrix Output + permissions: {} runs-on: ubuntu-latest outputs: images: ${{ steps.step1.outputs.matrix }}