Skip to content

KAFKA-19686 : release - Trigger docker builds with rc and release #21963

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
muralibasani wants to merge 4 commits into
base: trunk
Choose a base branch
from
+815 −2
Open

KAFKA-19686 : release - Trigger docker builds with rc and release #21963

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
120ced3
Trigger docker builds with rc and release
muralibasani Apr 4, 2026
ad95170
Adding tests and log statements
muralibasani Apr 8, 2026
855a4c1
From review
muralibasani Apr 12, 2026
cb8301b
Updating instruction for token reset
muralibasani Apr 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 128 additions & 0 deletions release/github.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Auxiliary functions to interact with the GitHub REST API.

Set the GITHUB_REPO environment variable to override the target repository
(e.g. "myuser/kafka" to test against a personal fork).

Set GITHUB_DRY_RUN=true to print API calls without executing them.
"""

import json
import os
import time
import urllib.request

from runtime import fail

GITHUB_API_URL = "https://api.github.com"
GITHUB_REPO = os.environ.get("GITHUB_REPO", "apache/kafka")
DRY_RUN = os.environ.get("GITHUB_DRY_RUN", "").lower() in ("true", "1", "yes")


def _api_request(token, method, path, body=None):
"""
Make an authenticated request to the GitHub REST API.
In dry-run mode, prints the request details without executing.
"""
url = f"{GITHUB_API_URL}{path}"

if DRY_RUN:
print(f"[DRY RUN] {method} {url}")
if body:
print(f"[DRY RUN] Body: {json.dumps(body, indent=2)}")
return None

data = json.dumps(body).encode("utf-8") if body else None
req = urllib.request.Request(url, data=data, method=method)
req.add_header("Accept", "application/vnd.github.v3+json")
req.add_header("Authorization", f"token {token}")
if data:
req.add_header("Content-Type", "application/json")
try:
with urllib.request.urlopen(req) as resp:
if resp.status == 204:
return None
return json.loads(resp.read().decode("utf-8"))
except urllib.error.HTTPError as e:
error_body = e.read().decode("utf-8") if e.fp else ""
fail(f"GitHub API error {e.code} for {method} {path}: {error_body}")


def _get_latest_run_url(token, workflow_file):
"""
Fetch the most recent run URL for a workflow. Returns the HTML URL
of the latest run, or a fallback URL to the workflow's runs page.
"""
fallback = f"https://github.com/{GITHUB_REPO}/actions/workflows/{workflow_file}"
path = f"/repos/{GITHUB_REPO}/actions/workflows/{workflow_file}/runs?per_page=1"
try:
result = _api_request(token, "GET", path)
if result and result.get("workflow_runs"):
return result["workflow_runs"][0]["html_url"]
except Exception:
pass
return fallback


def trigger_workflow(token, workflow_file, ref, inputs):
"""
Trigger a GitHub Actions workflow_dispatch event.
Returns None on success (HTTP 204).
"""
path = f"/repos/{GITHUB_REPO}/actions/workflows/{workflow_file}/dispatches"
body = {"ref": ref, "inputs": inputs}
print(f"Triggering workflow {workflow_file} on {GITHUB_REPO} with inputs: {json.dumps(inputs)}")
_api_request(token, "POST", path, body)
print(f"Successfully triggered {workflow_file}")
# Brief pause to allow GitHub to register the run before querying
if not DRY_RUN:
time.sleep(2)
run_url = _get_latest_run_url(token, workflow_file)
print(f" View run: {run_url}")


def trigger_docker_build_test(token, ref, image_type, kafka_url):
"""
Trigger the Docker Build Test workflow for the given image type.
"""
print(f"\n--- Docker Build Test ({image_type}) ---")
print(f" Image type : {image_type}")
print(f" Branch/ref : {ref}")
print(f" Kafka URL : {kafka_url}")
trigger_workflow(token, "docker_build_and_test.yml", ref, {
"image_type": image_type,
"kafka_url": kafka_url,
})


def trigger_docker_rc_release(token, ref, image_type, rc_docker_image, kafka_url):
"""
Trigger the Docker RC Release workflow for the given image type.
"""
print(f"\n--- Docker RC Release ({image_type}) ---")
print(f" Image type : {image_type}")
print(f" Docker image : {rc_docker_image}")
print(f" Branch/ref : {ref}")
print(f" Kafka URL : {kafka_url}")
trigger_workflow(token, "docker_rc_release.yml", ref, {
"image_type": image_type,
"rc_docker_image": rc_docker_image,
"kafka_url": kafka_url,
})
51 changes: 51 additions & 0 deletions release/release.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
repo_dir,
)
import git
import github
import gpg
import notes
import preferences
Expand Down Expand Up @@ -217,6 +218,54 @@ def command_release_announcement_email():
## Default 'stage' subcommand implementation isn't isolated to its own function yet for historical reasons


def trigger_docker_workflows(rc_tag, release_version, dev_branch):
"""
Trigger Docker image build/test and RC release workflows via GitHub Actions API.
Prompts the user for confirmation before each step.
"""
print("\n=== Docker Image Workflows ===")
if github.DRY_RUN:
print("NOTE: GITHUB_DRY_RUN is enabled. No actual API calls will be made.")
if github.GITHUB_REPO != "apache/kafka":
print(f"NOTE: Using custom repository: {github.GITHUB_REPO}")
if not confirm("Trigger Docker image build workflows via GitHub Actions?"):
print("Skipping Docker image workflows.")
return

def get_github_token():
print(templates.github_token_instructions())
return prompt("Enter your GitHub personal access token: ")
github_token = preferences.get('github_token', get_github_token)
kafka_url = f"https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz"

# Step 1: Trigger build/test workflows and loop until CVE-free
while True:
print(f"\nStep 1/2: Triggering Docker Build Test workflows for JVM and native images...")
for image_type in ["jvm", "native"]:
github.trigger_docker_build_test(github_token, dev_branch, image_type, kafka_url)
print("\nDocker Build Test workflows triggered successfully for both JVM and native images.")
print(f"\nPlease check the build results and CVE scan reports at:")
print(f" https://github.com/{github.GITHUB_REPO}/actions/workflows/docker_build_and_test.yml")
print("\nVerify that:")
print(" 1. Both JVM and native image builds succeeded")
print(" 2. The CVE scan reports show no CRITICAL or HIGH vulnerabilities")
print(" 3. If CVEs are found, update the Dockerfiles and re-trigger")
print(" Dockerfiles are located at: docker/jvm/Dockerfile and docker/native/Dockerfile")
if confirm("Have the builds passed with no CVEs? (n to re-trigger after fixing Dockerfiles)"):
break
print("\nRe-triggering Docker Build Test workflows after Dockerfile updates...")

# Step 2: Push RC images to DockerHub
print(f"\nStep 2/2: Triggering Docker RC Release workflows for JVM and native images...")
for image_type in ["jvm", "native"]:
docker_image_name = "apache/kafka-native" if image_type == "native" else "apache/kafka"
rc_docker_image = f"{docker_image_name}:{rc_tag}"
github.trigger_docker_rc_release(github_token, dev_branch, image_type, rc_docker_image, kafka_url)
print("\nDocker RC Release workflows triggered successfully for both JVM and native images.")

print(f"\nAll Docker workflow runs can be monitored at: https://github.com/{github.GITHUB_REPO}/actions")


def verify_gpg_key():
if not gpg.key_exists(gpg_key_id):
fail(f"GPG key {gpg_key_id} not found")
Expand Down Expand Up @@ -372,6 +421,8 @@ def delete_gitrefs():
git.push_ref(rc_tag)
git.push_ref(starting_branch)

trigger_docker_workflows(rc_tag, release_version, dev_branch)

# Move back to starting branch and clean out the temporary release branch (e.g. 1.0.0) we used to generate everything
git.reset_hard_head()
git.switch_branch(starting_branch)
Expand Down
26 changes: 24 additions & 2 deletions release/templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,9 @@ def deploy_instructions():
There will be more than one repository entries created, please close all of them.
In some cases, you may get errors on some repositories while closing them, see KAFKA-15033.
If this is not the first RC, you need to 'Drop' the previous artifacts.
Confirm the correct artifacts are visible at https://repository.apache.org/content/groups/staging/org/apache/kafka/ and build the
jvm and native Docker images following these instructions: https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=34840886#ReleaseProcess-CreateJVMApacheKafkaDockerArtifacts(Forversions>=3.7.0)
Confirm the correct artifacts are visible at https://repository.apache.org/content/groups/staging/org/apache/kafka/
Note: Docker image builds are triggered automatically by this script after the RC tag is pushed.
Monitor the workflow runs at https://github.com/apache/kafka/actions
"""

def sanity_check_instructions(release_version, rc_tag):
Expand Down Expand Up @@ -267,6 +268,27 @@ def rc_email_instructions(rc_email_text):
"""


def github_token_instructions():
return """
To trigger Docker image builds, you need a GitHub Personal Access Token.

How to generate one:
1. Go to https://github.com/settings/tokens
2. Click "Generate new token" -> "Generate new token (classic)"
3. Set a name (e.g. "kafka-release")
4. Set an expiration (e.g. 7 days is sufficient for a release cycle)
5. Select the scope: "repo" (Full control of private repositories)
- This includes the required "actions" write permission
6. Click "Generate token"
7. Copy the token (starts with "ghp_...")

Note: The token will be saved in your release preferences file (.release-settings.json) for reuse.
You only need to generate it once per release cycle.
To reset the saved token, remove the "github_token" entry from .release-settings.json
or delete the file entirely (this will clear all saved preferences).
"""


def cmd_failed():
return """
*************************************************
Expand Down
104 changes: 104 additions & 0 deletions release/test_docker_trigger_interactive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Interactive test script for the Docker workflow trigger flow.

This invokes the actual trigger_docker_workflows() function from release.py
without needing GPG, SVN, Maven, or committer access. The function is
extracted from release.py without executing its top-level interactive code.

Usage:
# Dry-run (no API calls, no token needed — recommended for first test):
GITHUB_DRY_RUN=true python test_docker_trigger_interactive.py

# Against your fork (real API calls, needs a GitHub token):
GITHUB_REPO=yourusername/kafka python test_docker_trigger_interactive.py

# Combine both:
GITHUB_DRY_RUN=true GITHUB_REPO=yourusername/kafka python test_docker_trigger_interactive.py
"""

import os
import sys

# Ensure release/ is on the path
sys.path.insert(0, os.path.dirname(__file__))

from runtime import confirm, confirm_or_fail, prompt
import github
import preferences
import templates


def _load_trigger_docker_workflows():
"""
Extract trigger_docker_workflows from release.py without executing the
module's top-level interactive code. We parse the source and compile just
the function definition, then bind it to real (not mocked) dependencies.
"""
release_path = os.path.join(os.path.dirname(__file__), "release.py")
with open(release_path) as f:
source = f.read()

lines = source.split('\n')
func_lines = []
capturing = False
for line in lines:
if line.startswith('def trigger_docker_workflows('):
capturing = True
elif capturing and line and not line[0].isspace() and not line.startswith('#'):
break
if capturing:
func_lines.append(line)

func_source = '\n'.join(func_lines)

ns = {
'github': github,
'confirm': confirm,
'confirm_or_fail': confirm_or_fail,
'preferences': preferences,
'templates': templates,
'prompt': prompt,
}
exec(compile(func_source, release_path, 'exec'), ns)
return ns['trigger_docker_workflows']


if __name__ == "__main__":
trigger_docker_workflows = _load_trigger_docker_workflows()

print("=" * 70)
print(" Docker Workflow Trigger - Interactive Test")
print("=" * 70)
print(f"\n Target repo : {github.GITHUB_REPO}")
print(f" Dry-run mode : {github.DRY_RUN}")
print()

release_version = prompt("Enter release version (e.g. 4.3.0): ")
rc = prompt("Enter RC number (e.g. 0): ")
rc_tag = f"{release_version}-rc{rc}"
dev_branch = '.'.join(release_version.split('.')[:2])

print(f"\n Release version : {release_version}")
print(f" RC tag : {rc_tag}")
print(f" Dev branch : {dev_branch}")

trigger_docker_workflows(rc_tag, release_version, dev_branch)

print("\nDone.")
Loading
Loading