From 120ced38fb3ea738372da3897325e2a5c8e35b8e Mon Sep 17 00:00:00 2001 From: Muralidhar Basani Date: Sat, 4 Apr 2026 10:45:31 +0200 Subject: [PATCH 1/4] Trigger docker builds with rc and release --- release/github.py | 82 ++++++++++++++++++++++++++++++++++++++++++++ release/release.py | 14 ++++++++ release/templates.py | 5 +-- 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 release/github.py diff --git a/release/github.py b/release/github.py new file mode 100644 index 0000000000000..8971d54382047 --- /dev/null +++ b/release/github.py @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Auxiliary functions to interact with the GitHub REST API. +""" + +import json +import urllib.request + +from runtime import fail + +GITHUB_API_URL = "https://api.github.com" +GITHUB_REPO = "apache/kafka" + + +def _api_request(token, method, path, body=None): + """ + Make an authenticated request to the GitHub REST API. + """ + url = f"{GITHUB_API_URL}{path}" + data = json.dumps(body).encode("utf-8") if body else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Accept", "application/vnd.github.v3+json") + req.add_header("Authorization", f"token {token}") + if data: + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req) as resp: + if resp.status == 204: + return None + return json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + error_body = e.read().decode("utf-8") if e.fp else "" + fail(f"GitHub API error {e.code} for {method} {path}: {error_body}") + + +def trigger_workflow(token, workflow_file, ref, inputs): + """ + Trigger a GitHub Actions workflow_dispatch event. + Returns None on success (HTTP 204). + """ + path = f"/repos/{GITHUB_REPO}/actions/workflows/{workflow_file}/dispatches" + body = {"ref": ref, "inputs": inputs} + print(f"Triggering workflow {workflow_file} with inputs: {json.dumps(inputs)}") + _api_request(token, "POST", path, body) + print(f"Successfully triggered {workflow_file}") + + +def trigger_docker_build_test(token, ref, image_type, kafka_url): + """ + Trigger the Docker Build Test workflow for the given image type. + """ + trigger_workflow(token, "docker_build_and_test.yml", ref, { + "image_type": image_type, + "kafka_url": kafka_url, + }) + + +def trigger_docker_rc_release(token, ref, image_type, rc_docker_image, kafka_url): + """ + Trigger the Docker RC Release workflow for the given image type. + """ + trigger_workflow(token, "docker_rc_release.yml", ref, { + "image_type": image_type, + "rc_docker_image": rc_docker_image, + "kafka_url": kafka_url, + }) diff --git a/release/release.py b/release/release.py index 7baf25a1b5301..9afbfc251388d 100644 --- a/release/release.py +++ b/release/release.py @@ -67,6 +67,7 @@ repo_dir, ) import git +import github import gpg import notes import preferences @@ -372,6 +373,19 @@ def delete_gitrefs(): git.push_ref(rc_tag) git.push_ref(starting_branch) +# Trigger Docker image build and test workflows via GitHub Actions +if confirm("Trigger Docker image build workflows via GitHub Actions?"): + github_token = preferences.get('github_token', lambda: prompt("Enter your GitHub personal access token (with 'actions' scope): ")) + kafka_url = f"https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz" + for image_type in ["jvm", "native"]: + github.trigger_docker_build_test(github_token, dev_branch, image_type, kafka_url) + if confirm("Also trigger Docker RC release workflows to push RC images to DockerHub?"): + for image_type in ["jvm", "native"]: + docker_image_name = "apache/kafka-native" if image_type == "native" else "apache/kafka" + rc_docker_image = f"{docker_image_name}:{rc_tag}" + github.trigger_docker_rc_release(github_token, dev_branch, image_type, rc_docker_image, kafka_url) + print(f"\nDocker workflow runs can be monitored at: https://github.com/apache/kafka/actions") + # Move back to starting branch and clean out the temporary release branch (e.g. 1.0.0) we used to generate everything git.reset_hard_head() git.switch_branch(starting_branch) diff --git a/release/templates.py b/release/templates.py index aff2d33cb91d2..f0f87cea2a15f 100644 --- a/release/templates.py +++ b/release/templates.py @@ -154,8 +154,9 @@ def deploy_instructions(): There will be more than one repository entries created, please close all of them. In some cases, you may get errors on some repositories while closing them, see KAFKA-15033. If this is not the first RC, you need to 'Drop' the previous artifacts. -Confirm the correct artifacts are visible at https://repository.apache.org/content/groups/staging/org/apache/kafka/ and build the -jvm and native Docker images following these instructions: https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=34840886#ReleaseProcess-CreateJVMApacheKafkaDockerArtifacts(Forversions>=3.7.0) +Confirm the correct artifacts are visible at https://repository.apache.org/content/groups/staging/org/apache/kafka/ +Note: Docker image builds are triggered automatically by this script after the RC tag is pushed. +Monitor the workflow runs at https://github.com/apache/kafka/actions """ def sanity_check_instructions(release_version, rc_tag): From ad9517009bca4ed0d124d0a9fdd76686658f5cf4 Mon Sep 17 00:00:00 2001 From: Muralidhar Basani Date: Wed, 8 Apr 2026 17:13:33 +0200 Subject: [PATCH 2/4] Adding tests and log statements --- release/github.py | 28 +++- release/release.py | 13 +- release/test_github.py | 331 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 369 insertions(+), 3 deletions(-) create mode 100644 release/test_github.py diff --git a/release/github.py b/release/github.py index 8971d54382047..fa4b7433f7efe 100644 --- a/release/github.py +++ b/release/github.py @@ -17,22 +17,37 @@ """ Auxiliary functions to interact with the GitHub REST API. + +Set the GITHUB_REPO environment variable to override the target repository +(e.g. "myuser/kafka" to test against a personal fork). + +Set GITHUB_DRY_RUN=true to print API calls without executing them. """ import json +import os import urllib.request from runtime import fail GITHUB_API_URL = "https://api.github.com" -GITHUB_REPO = "apache/kafka" +GITHUB_REPO = os.environ.get("GITHUB_REPO", "apache/kafka") +DRY_RUN = os.environ.get("GITHUB_DRY_RUN", "").lower() in ("true", "1", "yes") def _api_request(token, method, path, body=None): """ Make an authenticated request to the GitHub REST API. + In dry-run mode, prints the request details without executing. """ url = f"{GITHUB_API_URL}{path}" + + if DRY_RUN: + print(f"[DRY RUN] {method} {url}") + if body: + print(f"[DRY RUN] Body: {json.dumps(body, indent=2)}") + return None + data = json.dumps(body).encode("utf-8") if body else None req = urllib.request.Request(url, data=data, method=method) req.add_header("Accept", "application/vnd.github.v3+json") @@ -56,7 +71,7 @@ def trigger_workflow(token, workflow_file, ref, inputs): """ path = f"/repos/{GITHUB_REPO}/actions/workflows/{workflow_file}/dispatches" body = {"ref": ref, "inputs": inputs} - print(f"Triggering workflow {workflow_file} with inputs: {json.dumps(inputs)}") + print(f"Triggering workflow {workflow_file} on {GITHUB_REPO} with inputs: {json.dumps(inputs)}") _api_request(token, "POST", path, body) print(f"Successfully triggered {workflow_file}") @@ -65,6 +80,10 @@ def trigger_docker_build_test(token, ref, image_type, kafka_url): """ Trigger the Docker Build Test workflow for the given image type. """ + print(f"\n--- Docker Build Test ({image_type}) ---") + print(f" Image type : {image_type}") + print(f" Branch/ref : {ref}") + print(f" Kafka URL : {kafka_url}") trigger_workflow(token, "docker_build_and_test.yml", ref, { "image_type": image_type, "kafka_url": kafka_url, @@ -75,6 +94,11 @@ def trigger_docker_rc_release(token, ref, image_type, rc_docker_image, kafka_url """ Trigger the Docker RC Release workflow for the given image type. """ + print(f"\n--- Docker RC Release ({image_type}) ---") + print(f" Image type : {image_type}") + print(f" Docker image : {rc_docker_image}") + print(f" Branch/ref : {ref}") + print(f" Kafka URL : {kafka_url}") trigger_workflow(token, "docker_rc_release.yml", ref, { "image_type": image_type, "rc_docker_image": rc_docker_image, diff --git a/release/release.py b/release/release.py index 9afbfc251388d..b46f14ff6fcf1 100644 --- a/release/release.py +++ b/release/release.py @@ -374,17 +374,28 @@ def delete_gitrefs(): git.push_ref(starting_branch) # Trigger Docker image build and test workflows via GitHub Actions +print("\n=== Docker Image Workflows ===") +if github.DRY_RUN: + print("NOTE: GITHUB_DRY_RUN is enabled. No actual API calls will be made.") +if github.GITHUB_REPO != "apache/kafka": + print(f"NOTE: Using custom repository: {github.GITHUB_REPO}") if confirm("Trigger Docker image build workflows via GitHub Actions?"): github_token = preferences.get('github_token', lambda: prompt("Enter your GitHub personal access token (with 'actions' scope): ")) kafka_url = f"https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz" + print(f"\nStep 1/2: Triggering Docker Build Test workflows for JVM and native images...") for image_type in ["jvm", "native"]: github.trigger_docker_build_test(github_token, dev_branch, image_type, kafka_url) + print("\nDocker Build Test workflows triggered successfully for both JVM and native images.") if confirm("Also trigger Docker RC release workflows to push RC images to DockerHub?"): + print(f"\nStep 2/2: Triggering Docker RC Release workflows for JVM and native images...") for image_type in ["jvm", "native"]: docker_image_name = "apache/kafka-native" if image_type == "native" else "apache/kafka" rc_docker_image = f"{docker_image_name}:{rc_tag}" github.trigger_docker_rc_release(github_token, dev_branch, image_type, rc_docker_image, kafka_url) - print(f"\nDocker workflow runs can be monitored at: https://github.com/apache/kafka/actions") + print("\nDocker RC Release workflows triggered successfully for both JVM and native images.") + print(f"\nAll Docker workflow runs can be monitored at: https://github.com/{github.GITHUB_REPO}/actions") +else: + print("Skipping Docker image workflows.") # Move back to starting branch and clean out the temporary release branch (e.g. 1.0.0) we used to generate everything git.reset_hard_head() diff --git a/release/test_github.py b/release/test_github.py new file mode 100644 index 0000000000000..b783125d3d892 --- /dev/null +++ b/release/test_github.py @@ -0,0 +1,331 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Unit tests for the github module. +Run with: python -m pytest release/test_github.py -v + or: cd release && python -m pytest test_github.py -v +""" + +import json +import urllib.error +import unittest +from unittest.mock import patch, MagicMock + +import github + + +class TestApiRequest(unittest.TestCase): + + def _mock_response(self, status=204, body=None): + mock_resp = MagicMock() + mock_resp.status = status + mock_resp.read.return_value = json.dumps(body).encode("utf-8") if body else b"" + mock_resp.__enter__ = MagicMock(return_value=mock_resp) + mock_resp.__exit__ = MagicMock(return_value=False) + return mock_resp + + @patch("urllib.request.urlopen") + def test_post_request_204_returns_none(self, mock_urlopen): + mock_urlopen.return_value = self._mock_response(status=204) + + result = github._api_request("my-token", "POST", "/test/path", {"key": "val"}) + + self.assertIsNone(result) + req = mock_urlopen.call_args[0][0] + self.assertEqual(req.full_url, "https://api.github.com/test/path") + self.assertEqual(req.get_method(), "POST") + + @patch("urllib.request.urlopen") + def test_get_request_200_returns_json(self, mock_urlopen): + mock_urlopen.return_value = self._mock_response(status=200, body={"id": 42}) + + result = github._api_request("my-token", "GET", "/repos/test") + + self.assertEqual(result, {"id": 42}) + + @patch("urllib.request.urlopen") + def test_request_sets_auth_header(self, mock_urlopen): + mock_urlopen.return_value = self._mock_response(status=204) + + github._api_request("secret-token", "POST", "/path", {"a": 1}) + + req = mock_urlopen.call_args[0][0] + self.assertEqual(req.get_header("Authorization"), "token secret-token") + + @patch("urllib.request.urlopen") + def test_request_sets_accept_header(self, mock_urlopen): + mock_urlopen.return_value = self._mock_response(status=204) + + github._api_request("tok", "POST", "/path", {"a": 1}) + + req = mock_urlopen.call_args[0][0] + self.assertEqual(req.get_header("Accept"), "application/vnd.github.v3+json") + + @patch("urllib.request.urlopen") + def test_request_sets_content_type_when_body_present(self, mock_urlopen): + mock_urlopen.return_value = self._mock_response(status=204) + + github._api_request("tok", "POST", "/path", {"a": 1}) + + req = mock_urlopen.call_args[0][0] + self.assertEqual(req.get_header("Content-type"), "application/json") + + @patch("urllib.request.urlopen") + def test_request_no_content_type_when_no_body(self, mock_urlopen): + mock_urlopen.return_value = self._mock_response(status=200, body={"ok": True}) + + github._api_request("tok", "GET", "/path") + + req = mock_urlopen.call_args[0][0] + self.assertIsNone(req.get_header("Content-type")) + self.assertIsNone(req.data) + + @patch("urllib.request.urlopen") + def test_request_serializes_body_as_json(self, mock_urlopen): + mock_urlopen.return_value = self._mock_response(status=204) + + github._api_request("tok", "POST", "/path", {"ref": "main", "inputs": {"k": "v"}}) + + req = mock_urlopen.call_args[0][0] + self.assertEqual(json.loads(req.data), {"ref": "main", "inputs": {"k": "v"}}) + + @patch("github.fail") + @patch("urllib.request.urlopen") + def test_http_error_calls_fail(self, mock_urlopen, mock_fail): + error_body = b'{"message": "Not Found"}' + mock_fp = MagicMock() + mock_fp.read.return_value = error_body + http_error = urllib.error.HTTPError( + url="https://api.github.com/test", + code=404, + msg="Not Found", + hdrs={}, + fp=mock_fp, + ) + mock_urlopen.side_effect = http_error + + github._api_request("tok", "GET", "/test") + + mock_fail.assert_called_once() + fail_msg = mock_fail.call_args[0][0] + self.assertIn("404", fail_msg) + self.assertIn("GET", fail_msg) + self.assertIn("/test", fail_msg) + + +class TestDryRun(unittest.TestCase): + + def setUp(self): + self._orig_dry_run = github.DRY_RUN + self._orig_repo = github.GITHUB_REPO + + def tearDown(self): + github.DRY_RUN = self._orig_dry_run + github.GITHUB_REPO = self._orig_repo + + @patch("urllib.request.urlopen") + def test_dry_run_skips_http_call(self, mock_urlopen): + github.DRY_RUN = True + + result = github._api_request("tok", "POST", "/test", {"key": "val"}) + + self.assertIsNone(result) + mock_urlopen.assert_not_called() + + @patch("urllib.request.urlopen") + def test_dry_run_false_makes_http_call(self, mock_urlopen): + github.DRY_RUN = False + mock_resp = MagicMock() + mock_resp.status = 204 + mock_resp.__enter__ = MagicMock(return_value=mock_resp) + mock_resp.__exit__ = MagicMock(return_value=False) + mock_urlopen.return_value = mock_resp + + github._api_request("tok", "POST", "/test", {"key": "val"}) + + mock_urlopen.assert_called_once() + + +class TestConfigurableRepo(unittest.TestCase): + + def setUp(self): + self._orig_repo = github.GITHUB_REPO + + def tearDown(self): + github.GITHUB_REPO = self._orig_repo + + @patch("github._api_request") + def test_custom_repo_in_workflow_path(self, mock_api): + github.GITHUB_REPO = "myuser/kafka-fork" + + github.trigger_workflow("tok", "test.yml", "main", {"k": "v"}) + + path = mock_api.call_args[0][2] + self.assertIn("myuser/kafka-fork", path) + self.assertNotIn("apache/kafka", path) + + @patch("github._api_request") + def test_default_repo_is_apache_kafka(self, mock_api): + github.GITHUB_REPO = "apache/kafka" + + github.trigger_workflow("tok", "test.yml", "main", {"k": "v"}) + + path = mock_api.call_args[0][2] + self.assertIn("apache/kafka", path) + + +class TestTriggerWorkflow(unittest.TestCase): + + @patch("github._api_request") + def test_trigger_workflow_calls_correct_endpoint(self, mock_api): + github.trigger_workflow("tok", "my_workflow.yml", "main", {"key": "val"}) + + mock_api.assert_called_once_with( + "tok", "POST", + f"/repos/{github.GITHUB_REPO}/actions/workflows/my_workflow.yml/dispatches", + {"ref": "main", "inputs": {"key": "val"}}, + ) + + +class TestTriggerDockerBuildTest(unittest.TestCase): + + @patch("github._api_request") + def test_jvm_image(self, mock_api): + github.trigger_docker_build_test("tok", "4.3", "jvm", "https://example.com/kafka.tgz") + + mock_api.assert_called_once_with( + "tok", "POST", + "/repos/apache/kafka/actions/workflows/docker_build_and_test.yml/dispatches", + {"ref": "4.3", "inputs": {"image_type": "jvm", "kafka_url": "https://example.com/kafka.tgz"}}, + ) + + @patch("github._api_request") + def test_native_image(self, mock_api): + github.trigger_docker_build_test("tok", "4.3", "native", "https://example.com/kafka.tgz") + + mock_api.assert_called_once_with( + "tok", "POST", + "/repos/apache/kafka/actions/workflows/docker_build_and_test.yml/dispatches", + {"ref": "4.3", "inputs": {"image_type": "native", "kafka_url": "https://example.com/kafka.tgz"}}, + ) + + +class TestTriggerDockerRcRelease(unittest.TestCase): + + @patch("github._api_request") + def test_jvm_rc_release(self, mock_api): + github.trigger_docker_rc_release( + "tok", "4.3", "jvm", "apache/kafka:4.3.0-rc0", "https://example.com/kafka.tgz" + ) + + mock_api.assert_called_once_with( + "tok", "POST", + "/repos/apache/kafka/actions/workflows/docker_rc_release.yml/dispatches", + {"ref": "4.3", "inputs": { + "image_type": "jvm", + "rc_docker_image": "apache/kafka:4.3.0-rc0", + "kafka_url": "https://example.com/kafka.tgz", + }}, + ) + + @patch("github._api_request") + def test_native_rc_release(self, mock_api): + github.trigger_docker_rc_release( + "tok", "4.3", "native", "apache/kafka-native:4.3.0-rc0", "https://example.com/kafka.tgz" + ) + + mock_api.assert_called_once_with( + "tok", "POST", + "/repos/apache/kafka/actions/workflows/docker_rc_release.yml/dispatches", + {"ref": "4.3", "inputs": { + "image_type": "native", + "rc_docker_image": "apache/kafka-native:4.3.0-rc0", + "kafka_url": "https://example.com/kafka.tgz", + }}, + ) + + +class TestWorkflowInputAlignment(unittest.TestCase): + """Verify that the inputs we send match what the workflow YAML files expect.""" + + def _load_workflow_inputs(self, workflow_file): + import yaml + import os + base = os.path.join(os.path.dirname(__file__), "..", ".github", "workflows") + with open(os.path.join(base, workflow_file)) as f: + wf = yaml.safe_load(f) + # PyYAML parses 'on' as boolean True + return set(wf[True]["workflow_dispatch"]["inputs"].keys()) + + def test_build_and_test_inputs_match(self): + expected = self._load_workflow_inputs("docker_build_and_test.yml") + sent = {"image_type", "kafka_url"} + self.assertEqual(sent, expected, + f"github.trigger_docker_build_test sends {sent} but workflow expects {expected}") + + def test_rc_release_inputs_match(self): + expected = self._load_workflow_inputs("docker_rc_release.yml") + sent = {"image_type", "rc_docker_image", "kafka_url"} + self.assertEqual(sent, expected, + f"github.trigger_docker_rc_release sends {sent} but workflow expects {expected}") + + +class TestReleaseScriptIntegration(unittest.TestCase): + """Simulate the exact flow that release.py uses to trigger Docker workflows.""" + + @patch("github._api_request") + def test_full_release_flow(self, mock_api): + release_version = "4.3.0" + rc_tag = "4.3.0-rc0" + dev_branch = "4.3" + kafka_url = f"https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz" + + # Step 1: Build & test for both image types (as release.py does) + for image_type in ["jvm", "native"]: + github.trigger_docker_build_test("tok", dev_branch, image_type, kafka_url) + + # Step 2: RC release for both image types (as release.py does) + for image_type in ["jvm", "native"]: + docker_image_name = "apache/kafka-native" if image_type == "native" else "apache/kafka" + rc_docker_image = f"{docker_image_name}:{rc_tag}" + github.trigger_docker_rc_release("tok", dev_branch, image_type, rc_docker_image, kafka_url) + + self.assertEqual(mock_api.call_count, 4) + + # Verify each call's workflow file and inputs + calls = mock_api.call_args_list + + # Build test JVM + self.assertIn("docker_build_and_test.yml", calls[0][0][2]) + self.assertEqual(calls[0][0][3]["inputs"]["image_type"], "jvm") + + # Build test native + self.assertIn("docker_build_and_test.yml", calls[1][0][2]) + self.assertEqual(calls[1][0][3]["inputs"]["image_type"], "native") + + # RC release JVM + self.assertIn("docker_rc_release.yml", calls[2][0][2]) + self.assertEqual(calls[2][0][3]["inputs"]["rc_docker_image"], "apache/kafka:4.3.0-rc0") + + # RC release native + self.assertIn("docker_rc_release.yml", calls[3][0][2]) + self.assertEqual(calls[3][0][3]["inputs"]["rc_docker_image"], "apache/kafka-native:4.3.0-rc0") + + +if __name__ == "__main__": + unittest.main() From 855a4c1b1d4aea5dfbf230d4f8253e17316e95c5 Mon Sep 17 00:00:00 2001 From: Muralidhar Basani Date: Sun, 12 Apr 2026 17:40:26 +0200 Subject: [PATCH 3/4] From review --- release/github.py | 22 ++ release/release.py | 72 ++++--- release/templates.py | 19 ++ release/test_docker_trigger_interactive.py | 104 ++++++++++ release/test_github.py | 221 +++++++++++++++++++-- 5 files changed, 393 insertions(+), 45 deletions(-) create mode 100644 release/test_docker_trigger_interactive.py diff --git a/release/github.py b/release/github.py index fa4b7433f7efe..8631eac530128 100644 --- a/release/github.py +++ b/release/github.py @@ -26,6 +26,7 @@ import json import os +import time import urllib.request from runtime import fail @@ -64,6 +65,22 @@ def _api_request(token, method, path, body=None): fail(f"GitHub API error {e.code} for {method} {path}: {error_body}") +def _get_latest_run_url(token, workflow_file): + """ + Fetch the most recent run URL for a workflow. Returns the HTML URL + of the latest run, or a fallback URL to the workflow's runs page. + """ + fallback = f"https://github.com/{GITHUB_REPO}/actions/workflows/{workflow_file}" + path = f"/repos/{GITHUB_REPO}/actions/workflows/{workflow_file}/runs?per_page=1" + try: + result = _api_request(token, "GET", path) + if result and result.get("workflow_runs"): + return result["workflow_runs"][0]["html_url"] + except Exception: + pass + return fallback + + def trigger_workflow(token, workflow_file, ref, inputs): """ Trigger a GitHub Actions workflow_dispatch event. @@ -74,6 +91,11 @@ def trigger_workflow(token, workflow_file, ref, inputs): print(f"Triggering workflow {workflow_file} on {GITHUB_REPO} with inputs: {json.dumps(inputs)}") _api_request(token, "POST", path, body) print(f"Successfully triggered {workflow_file}") + # Brief pause to allow GitHub to register the run before querying + if not DRY_RUN: + time.sleep(2) + run_url = _get_latest_run_url(token, workflow_file) + print(f" View run: {run_url}") def trigger_docker_build_test(token, ref, image_type, kafka_url): diff --git a/release/release.py b/release/release.py index b46f14ff6fcf1..fcd39ffb584ef 100644 --- a/release/release.py +++ b/release/release.py @@ -218,6 +218,54 @@ def command_release_announcement_email(): ## Default 'stage' subcommand implementation isn't isolated to its own function yet for historical reasons +def trigger_docker_workflows(rc_tag, release_version, dev_branch): + """ + Trigger Docker image build/test and RC release workflows via GitHub Actions API. + Prompts the user for confirmation before each step. + """ + print("\n=== Docker Image Workflows ===") + if github.DRY_RUN: + print("NOTE: GITHUB_DRY_RUN is enabled. No actual API calls will be made.") + if github.GITHUB_REPO != "apache/kafka": + print(f"NOTE: Using custom repository: {github.GITHUB_REPO}") + if not confirm("Trigger Docker image build workflows via GitHub Actions?"): + print("Skipping Docker image workflows.") + return + + def get_github_token(): + print(templates.github_token_instructions()) + return prompt("Enter your GitHub personal access token: ") + github_token = preferences.get('github_token', get_github_token) + kafka_url = f"https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz" + + # Step 1: Trigger build/test workflows and loop until CVE-free + while True: + print(f"\nStep 1/2: Triggering Docker Build Test workflows for JVM and native images...") + for image_type in ["jvm", "native"]: + github.trigger_docker_build_test(github_token, dev_branch, image_type, kafka_url) + print("\nDocker Build Test workflows triggered successfully for both JVM and native images.") + print(f"\nPlease check the build results and CVE scan reports at:") + print(f" https://github.com/{github.GITHUB_REPO}/actions/workflows/docker_build_and_test.yml") + print("\nVerify that:") + print(" 1. Both JVM and native image builds succeeded") + print(" 2. The CVE scan reports show no CRITICAL or HIGH vulnerabilities") + print(" 3. If CVEs are found, update the Dockerfiles and re-trigger") + print(" Dockerfiles are located at: docker/jvm/Dockerfile and docker/native/Dockerfile") + if confirm("Have the builds passed with no CVEs? (n to re-trigger after fixing Dockerfiles)"): + break + print("\nRe-triggering Docker Build Test workflows after Dockerfile updates...") + + # Step 2: Push RC images to DockerHub + print(f"\nStep 2/2: Triggering Docker RC Release workflows for JVM and native images...") + for image_type in ["jvm", "native"]: + docker_image_name = "apache/kafka-native" if image_type == "native" else "apache/kafka" + rc_docker_image = f"{docker_image_name}:{rc_tag}" + github.trigger_docker_rc_release(github_token, dev_branch, image_type, rc_docker_image, kafka_url) + print("\nDocker RC Release workflows triggered successfully for both JVM and native images.") + + print(f"\nAll Docker workflow runs can be monitored at: https://github.com/{github.GITHUB_REPO}/actions") + + def verify_gpg_key(): if not gpg.key_exists(gpg_key_id): fail(f"GPG key {gpg_key_id} not found") @@ -373,29 +421,7 @@ def delete_gitrefs(): git.push_ref(rc_tag) git.push_ref(starting_branch) -# Trigger Docker image build and test workflows via GitHub Actions -print("\n=== Docker Image Workflows ===") -if github.DRY_RUN: - print("NOTE: GITHUB_DRY_RUN is enabled. No actual API calls will be made.") -if github.GITHUB_REPO != "apache/kafka": - print(f"NOTE: Using custom repository: {github.GITHUB_REPO}") -if confirm("Trigger Docker image build workflows via GitHub Actions?"): - github_token = preferences.get('github_token', lambda: prompt("Enter your GitHub personal access token (with 'actions' scope): ")) - kafka_url = f"https://dist.apache.org/repos/dist/dev/kafka/{rc_tag}/kafka_2.13-{release_version}.tgz" - print(f"\nStep 1/2: Triggering Docker Build Test workflows for JVM and native images...") - for image_type in ["jvm", "native"]: - github.trigger_docker_build_test(github_token, dev_branch, image_type, kafka_url) - print("\nDocker Build Test workflows triggered successfully for both JVM and native images.") - if confirm("Also trigger Docker RC release workflows to push RC images to DockerHub?"): - print(f"\nStep 2/2: Triggering Docker RC Release workflows for JVM and native images...") - for image_type in ["jvm", "native"]: - docker_image_name = "apache/kafka-native" if image_type == "native" else "apache/kafka" - rc_docker_image = f"{docker_image_name}:{rc_tag}" - github.trigger_docker_rc_release(github_token, dev_branch, image_type, rc_docker_image, kafka_url) - print("\nDocker RC Release workflows triggered successfully for both JVM and native images.") - print(f"\nAll Docker workflow runs can be monitored at: https://github.com/{github.GITHUB_REPO}/actions") -else: - print("Skipping Docker image workflows.") +trigger_docker_workflows(rc_tag, release_version, dev_branch) # Move back to starting branch and clean out the temporary release branch (e.g. 1.0.0) we used to generate everything git.reset_hard_head() diff --git a/release/templates.py b/release/templates.py index f0f87cea2a15f..a43df1738fd72 100644 --- a/release/templates.py +++ b/release/templates.py @@ -268,6 +268,25 @@ def rc_email_instructions(rc_email_text): """ +def github_token_instructions(): + return """ +To trigger Docker image builds, you need a GitHub Personal Access Token. + +How to generate one: + 1. Go to https://github.com/settings/tokens + 2. Click "Generate new token" -> "Generate new token (classic)" + 3. Set a name (e.g. "kafka-release") + 4. Set an expiration (e.g. 7 days is sufficient for a release cycle) + 5. Select the scope: "repo" (Full control of private repositories) + - This includes the required "actions" write permission + 6. Click "Generate token" + 7. Copy the token (starts with "ghp_...") + +Note: The token will be saved in your release preferences file for reuse. + You only need to generate it once per release cycle. +""" + + def cmd_failed(): return """ ************************************************* diff --git a/release/test_docker_trigger_interactive.py b/release/test_docker_trigger_interactive.py new file mode 100644 index 0000000000000..da74f34dc68ff --- /dev/null +++ b/release/test_docker_trigger_interactive.py @@ -0,0 +1,104 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Interactive test script for the Docker workflow trigger flow. + +This invokes the actual trigger_docker_workflows() function from release.py +without needing GPG, SVN, Maven, or committer access. The function is +extracted from release.py without executing its top-level interactive code. + +Usage: + # Dry-run (no API calls, no token needed — recommended for first test): + GITHUB_DRY_RUN=true python test_docker_trigger_interactive.py + + # Against your fork (real API calls, needs a GitHub token): + GITHUB_REPO=yourusername/kafka python test_docker_trigger_interactive.py + + # Combine both: + GITHUB_DRY_RUN=true GITHUB_REPO=yourusername/kafka python test_docker_trigger_interactive.py +""" + +import os +import sys + +# Ensure release/ is on the path +sys.path.insert(0, os.path.dirname(__file__)) + +from runtime import confirm, confirm_or_fail, prompt +import github +import preferences +import templates + + +def _load_trigger_docker_workflows(): + """ + Extract trigger_docker_workflows from release.py without executing the + module's top-level interactive code. We parse the source and compile just + the function definition, then bind it to real (not mocked) dependencies. + """ + release_path = os.path.join(os.path.dirname(__file__), "release.py") + with open(release_path) as f: + source = f.read() + + lines = source.split('\n') + func_lines = [] + capturing = False + for line in lines: + if line.startswith('def trigger_docker_workflows('): + capturing = True + elif capturing and line and not line[0].isspace() and not line.startswith('#'): + break + if capturing: + func_lines.append(line) + + func_source = '\n'.join(func_lines) + + ns = { + 'github': github, + 'confirm': confirm, + 'confirm_or_fail': confirm_or_fail, + 'preferences': preferences, + 'templates': templates, + 'prompt': prompt, + } + exec(compile(func_source, release_path, 'exec'), ns) + return ns['trigger_docker_workflows'] + + +if __name__ == "__main__": + trigger_docker_workflows = _load_trigger_docker_workflows() + + print("=" * 70) + print(" Docker Workflow Trigger - Interactive Test") + print("=" * 70) + print(f"\n Target repo : {github.GITHUB_REPO}") + print(f" Dry-run mode : {github.DRY_RUN}") + print() + + release_version = prompt("Enter release version (e.g. 4.3.0): ") + rc = prompt("Enter RC number (e.g. 0): ") + rc_tag = f"{release_version}-rc{rc}" + dev_branch = '.'.join(release_version.split('.')[:2]) + + print(f"\n Release version : {release_version}") + print(f" RC tag : {rc_tag}") + print(f" Dev branch : {dev_branch}") + + trigger_docker_workflows(rc_tag, release_version, dev_branch) + + print("\nDone.") diff --git a/release/test_github.py b/release/test_github.py index b783125d3d892..3db65763c2ba1 100644 --- a/release/test_github.py +++ b/release/test_github.py @@ -189,17 +189,33 @@ def test_default_repo_is_apache_kafka(self, mock_api): self.assertIn("apache/kafka", path) +def _post_calls(mock_api): + """Filter mock_api calls to only POST (dispatch) calls, ignoring GET (run URL lookup) calls.""" + return [c for c in mock_api.call_args_list if c[0][1] == "POST"] + + class TestTriggerWorkflow(unittest.TestCase): @patch("github._api_request") def test_trigger_workflow_calls_correct_endpoint(self, mock_api): github.trigger_workflow("tok", "my_workflow.yml", "main", {"key": "val"}) - mock_api.assert_called_once_with( + posts = _post_calls(mock_api) + self.assertEqual(len(posts), 1) + self.assertEqual(posts[0][0], ( "tok", "POST", f"/repos/{github.GITHUB_REPO}/actions/workflows/my_workflow.yml/dispatches", {"ref": "main", "inputs": {"key": "val"}}, - ) + )) + + @patch("github._api_request") + def test_trigger_workflow_fetches_run_url(self, mock_api): + """Verify that a GET call is made to fetch the latest run URL.""" + github.trigger_workflow("tok", "my_workflow.yml", "main", {"key": "val"}) + + get_calls = [c for c in mock_api.call_args_list if c[0][1] == "GET"] + self.assertEqual(len(get_calls), 1) + self.assertIn("/runs?per_page=1", get_calls[0][0][2]) class TestTriggerDockerBuildTest(unittest.TestCase): @@ -208,21 +224,25 @@ class TestTriggerDockerBuildTest(unittest.TestCase): def test_jvm_image(self, mock_api): github.trigger_docker_build_test("tok", "4.3", "jvm", "https://example.com/kafka.tgz") - mock_api.assert_called_once_with( + posts = _post_calls(mock_api) + self.assertEqual(len(posts), 1) + self.assertEqual(posts[0][0], ( "tok", "POST", "/repos/apache/kafka/actions/workflows/docker_build_and_test.yml/dispatches", {"ref": "4.3", "inputs": {"image_type": "jvm", "kafka_url": "https://example.com/kafka.tgz"}}, - ) + )) @patch("github._api_request") def test_native_image(self, mock_api): github.trigger_docker_build_test("tok", "4.3", "native", "https://example.com/kafka.tgz") - mock_api.assert_called_once_with( + posts = _post_calls(mock_api) + self.assertEqual(len(posts), 1) + self.assertEqual(posts[0][0], ( "tok", "POST", "/repos/apache/kafka/actions/workflows/docker_build_and_test.yml/dispatches", {"ref": "4.3", "inputs": {"image_type": "native", "kafka_url": "https://example.com/kafka.tgz"}}, - ) + )) class TestTriggerDockerRcRelease(unittest.TestCase): @@ -233,7 +253,9 @@ def test_jvm_rc_release(self, mock_api): "tok", "4.3", "jvm", "apache/kafka:4.3.0-rc0", "https://example.com/kafka.tgz" ) - mock_api.assert_called_once_with( + posts = _post_calls(mock_api) + self.assertEqual(len(posts), 1) + self.assertEqual(posts[0][0], ( "tok", "POST", "/repos/apache/kafka/actions/workflows/docker_rc_release.yml/dispatches", {"ref": "4.3", "inputs": { @@ -241,7 +263,7 @@ def test_jvm_rc_release(self, mock_api): "rc_docker_image": "apache/kafka:4.3.0-rc0", "kafka_url": "https://example.com/kafka.tgz", }}, - ) + )) @patch("github._api_request") def test_native_rc_release(self, mock_api): @@ -249,7 +271,9 @@ def test_native_rc_release(self, mock_api): "tok", "4.3", "native", "apache/kafka-native:4.3.0-rc0", "https://example.com/kafka.tgz" ) - mock_api.assert_called_once_with( + posts = _post_calls(mock_api) + self.assertEqual(len(posts), 1) + self.assertEqual(posts[0][0], ( "tok", "POST", "/repos/apache/kafka/actions/workflows/docker_rc_release.yml/dispatches", {"ref": "4.3", "inputs": { @@ -257,7 +281,7 @@ def test_native_rc_release(self, mock_api): "rc_docker_image": "apache/kafka-native:4.3.0-rc0", "kafka_url": "https://example.com/kafka.tgz", }}, - ) + )) class TestWorkflowInputAlignment(unittest.TestCase): @@ -305,26 +329,179 @@ def test_full_release_flow(self, mock_api): rc_docker_image = f"{docker_image_name}:{rc_tag}" github.trigger_docker_rc_release("tok", dev_branch, image_type, rc_docker_image, kafka_url) - self.assertEqual(mock_api.call_count, 4) - - # Verify each call's workflow file and inputs - calls = mock_api.call_args_list + # 4 POST (dispatch) + 4 GET (run URL lookup) = 8 total + posts = _post_calls(mock_api) + self.assertEqual(len(posts), 4) # Build test JVM - self.assertIn("docker_build_and_test.yml", calls[0][0][2]) - self.assertEqual(calls[0][0][3]["inputs"]["image_type"], "jvm") + self.assertIn("docker_build_and_test.yml", posts[0][0][2]) + self.assertEqual(posts[0][0][3]["inputs"]["image_type"], "jvm") # Build test native - self.assertIn("docker_build_and_test.yml", calls[1][0][2]) - self.assertEqual(calls[1][0][3]["inputs"]["image_type"], "native") + self.assertIn("docker_build_and_test.yml", posts[1][0][2]) + self.assertEqual(posts[1][0][3]["inputs"]["image_type"], "native") # RC release JVM - self.assertIn("docker_rc_release.yml", calls[2][0][2]) - self.assertEqual(calls[2][0][3]["inputs"]["rc_docker_image"], "apache/kafka:4.3.0-rc0") + self.assertIn("docker_rc_release.yml", posts[2][0][2]) + self.assertEqual(posts[2][0][3]["inputs"]["rc_docker_image"], "apache/kafka:4.3.0-rc0") # RC release native - self.assertIn("docker_rc_release.yml", calls[3][0][2]) - self.assertEqual(calls[3][0][3]["inputs"]["rc_docker_image"], "apache/kafka-native:4.3.0-rc0") + self.assertIn("docker_rc_release.yml", posts[3][0][2]) + self.assertEqual(posts[3][0][3]["inputs"]["rc_docker_image"], "apache/kafka-native:4.3.0-rc0") + + +# We need to import trigger_docker_workflows from release.py, but that file +# executes interactively at module level. So we import it directly from the +# function definition using importlib to avoid running the top-level code. +def _load_trigger_docker_workflows(): + """ + Extract trigger_docker_workflows from release.py without executing the + module's top-level interactive code. We parse the source and compile just + the function definition. + """ + import os, types + release_path = os.path.join(os.path.dirname(__file__), "release.py") + with open(release_path) as f: + source = f.read() + + # Extract the function source (from 'def trigger_docker_workflows' to the + # next top-level def or non-indented statement) + lines = source.split('\n') + func_lines = [] + capturing = False + for line in lines: + if line.startswith('def trigger_docker_workflows('): + capturing = True + elif capturing and line and not line[0].isspace() and not line.startswith('#'): + break + if capturing: + func_lines.append(line) + + func_source = '\n'.join(func_lines) + + # Create a module-like namespace with the dependencies the function needs + ns = { + 'github': github, + 'confirm': None, # will be mocked per test + 'confirm_or_fail': None, # will be mocked per test + 'preferences': None, # will be mocked per test + 'templates': None, # will be mocked per test + 'prompt': None, # will be mocked per test + } + exec(compile(func_source, release_path, 'exec'), ns) + return ns['trigger_docker_workflows'], ns + + +_trigger_fn, _fn_namespace = _load_trigger_docker_workflows() + + +class TestTriggerDockerWorkflows(unittest.TestCase): + """Test the trigger_docker_workflows function from release.py.""" + + def setUp(self): + self._orig_dry_run = github.DRY_RUN + github.DRY_RUN = True # Always dry-run in tests + + def tearDown(self): + github.DRY_RUN = self._orig_dry_run + + def _run(self, confirm_responses, confirm_or_fail_responses=None): + """ + Run trigger_docker_workflows with mocked interactive prompts. + confirm_responses: list of booleans for each confirm() call + confirm_or_fail_responses: list of None (success) values for confirm_or_fail() + """ + confirm_iter = iter(confirm_responses) + _fn_namespace['confirm'] = lambda msg: next(confirm_iter) + _fn_namespace['confirm_or_fail'] = lambda msg: None # always succeeds + _fn_namespace['preferences'] = MagicMock() + _fn_namespace['preferences'].get = MagicMock(return_value="fake-token") + _fn_namespace['templates'] = MagicMock() + _fn_namespace['templates'].github_token_instructions = MagicMock(return_value="token instructions") + _fn_namespace['prompt'] = MagicMock(return_value="fake-token") + + with patch("github._api_request") as mock_api: + _trigger_fn("4.3.0-rc0", "4.3.0", "4.3") + return mock_api + + def test_happy_path_all_yes(self): + """User says yes to everything, builds pass first time.""" + # confirm calls: 1) trigger? yes, 2) builds passed? yes + mock_api = self._run([True, True]) + + posts = _post_calls(mock_api) + self.assertEqual(len(posts), 4) + + # First 2: build_and_test (jvm, native) + self.assertIn("docker_build_and_test.yml", posts[0][0][2]) + self.assertIn("docker_build_and_test.yml", posts[1][0][2]) + # Last 2: rc_release (jvm, native) + self.assertIn("docker_rc_release.yml", posts[2][0][2]) + self.assertIn("docker_rc_release.yml", posts[3][0][2]) + + def test_skip_docker_workflows(self): + """User declines to trigger Docker workflows.""" + # confirm calls: 1) trigger? no + mock_api = self._run([False]) + + mock_api.assert_not_called() + + def test_cve_retry_then_pass(self): + """CVEs found on first attempt, user retries, second attempt passes.""" + # confirm calls: 1) trigger? yes, 2) builds passed? no (CVE found), + # 3) builds passed? yes (after retry) + mock_api = self._run([True, False, True]) + + posts = _post_calls(mock_api) + # 2 build_test (1st) + 2 build_test (retry) + 2 rc_release = 6 + self.assertEqual(len(posts), 6) + + # First 4: build_and_test (2 attempts x 2 image types) + for i in range(4): + self.assertIn("docker_build_and_test.yml", posts[i][0][2]) + # Last 2: rc_release + self.assertIn("docker_rc_release.yml", posts[4][0][2]) + self.assertIn("docker_rc_release.yml", posts[5][0][2]) + + def test_multiple_cve_retries(self): + """CVEs found twice, third attempt passes.""" + # confirm calls: 1) trigger? yes, 2) passed? no, 3) passed? no, 4) passed? yes + mock_api = self._run([True, False, False, True]) + + posts = _post_calls(mock_api) + # 3 rounds of build_test (6) + 1 round of rc_release (2) = 8 + self.assertEqual(len(posts), 8) + + def test_rc_release_uses_correct_image_names(self): + """Verify JVM uses apache/kafka and native uses apache/kafka-native.""" + mock_api = self._run([True, True]) + + posts = _post_calls(mock_api) + # RC release JVM (3rd POST) + jvm_inputs = posts[2][0][3]["inputs"] + self.assertEqual(jvm_inputs["rc_docker_image"], "apache/kafka:4.3.0-rc0") + self.assertEqual(jvm_inputs["image_type"], "jvm") + + # RC release native (4th POST) + native_inputs = posts[3][0][3]["inputs"] + self.assertEqual(native_inputs["rc_docker_image"], "apache/kafka-native:4.3.0-rc0") + self.assertEqual(native_inputs["image_type"], "native") + + def test_kafka_url_construction(self): + """Verify the kafka_url is constructed correctly from rc_tag and release_version.""" + mock_api = self._run([True, True]) + + expected_url = "https://dist.apache.org/repos/dist/dev/kafka/4.3.0-rc0/kafka_2.13-4.3.0.tgz" + posts = _post_calls(mock_api) + self.assertEqual(posts[0][0][3]["inputs"]["kafka_url"], expected_url) + + def test_dev_branch_used_as_ref(self): + """Verify dev_branch is passed as the workflow ref.""" + mock_api = self._run([True, True]) + + posts = _post_calls(mock_api) + for post in posts: + self.assertEqual(post[0][3]["ref"], "4.3") if __name__ == "__main__": From cb8301b7feec3230113ae62b5d52d27b5ccbd7e2 Mon Sep 17 00:00:00 2001 From: Muralidhar Basani Date: Sun, 12 Apr 2026 18:03:31 +0200 Subject: [PATCH 4/4] Updating instruction for token reset --- release/templates.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/templates.py b/release/templates.py index a43df1738fd72..e59df30b56d05 100644 --- a/release/templates.py +++ b/release/templates.py @@ -282,8 +282,10 @@ def github_token_instructions(): 6. Click "Generate token" 7. Copy the token (starts with "ghp_...") -Note: The token will be saved in your release preferences file for reuse. +Note: The token will be saved in your release preferences file (.release-settings.json) for reuse. You only need to generate it once per release cycle. + To reset the saved token, remove the "github_token" entry from .release-settings.json + or delete the file entirely (this will clear all saved preferences). """