From 14cd1912237f614b66a53d897b42ab5e8e656a5c Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 28 Apr 2025 21:47:11 +0800 Subject: [PATCH 01/37] RADAS: add radas configurations --- charon/config.py | 69 +++++++++++ charon/schemas/charon.json | 38 +++++++ tests/test_config_radas.py | 226 +++++++++++++++++++++++++++++++++++++ 3 files changed, 333 insertions(+) create mode 100644 tests/test_config_radas.py diff --git a/charon/config.py b/charon/config.py index 86f826ea..35efe6eb 100644 --- a/charon/config.py +++ b/charon/config.py @@ -24,6 +24,69 @@ logger = logging.getLogger(__name__) +class RadasConfig(object): + def __init__(self, data: Dict): + self.__umb_host: str = data.get("umb_host", None) + self.__umb_host_port: str = data.get("umb_host_port", "5671") + self.__result_queue: str = data.get("result_queue", None) + self.__request_queue: str = data.get("request_queue", None) + self.__client_ca: str = data.get("client_ca", None) + self.__client_key: str = data.get("client_key", None) + self.__client_key_pass_file: str = data.get("client_key_pass_file", None) + self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + + def validate(self) -> bool: + if not self.__umb_host: + logger.error("Missing host name setting for UMB!") + return False + if not self.__result_queue: + logger.error("Missing the queue setting to receive siging result in UMB!") + return False + if not self.__request_queue: + logger.error("Missing the queue setting to send signing request in UMB!") + return False + if self.__client_ca and not os.access(self.__client_ca, os.R_OK): + logger.error("The client CA file is not valid!") + return False + if self.__client_key and not os.access(self.__client_key, os.R_OK): + logger.error("The client key file is not valid!") + return False + if self.__client_key_pass_file and not os.access(self.__client_key_pass_file, os.R_OK): + logger.error("The client key password file is not valid!") + return False + if self.__root_ca and not os.access(self.__root_ca, os.R_OK): + logger.error("The root ca file is not valid!") + return False + return True + + def umb_target(self) -> str: + return f'amqps://{self.__umb_host}:{self.__umb_host_port}' + + def result_queue(self) -> str: + return self.__result_queue + + def request_queue(self) -> str: + return self.__request_queue + + def client_ca(self) -> str: + return self.__client_ca + + def client_key(self) -> str: + return self.__client_key + + def client_key_password(self) -> str: + pass_file = self.__client_key_pass_file + if os.access(pass_file, os.R_OK): + with open(pass_file, 'r') as f: + return f.read() + elif pass_file: + logger.warning("The key password file is not accessible. Will ignore the password.") + return "" + + def root_ca(self) -> str: + return self.__root_ca + + class CharonConfig(object): """CharonConfig is used to store all configurations for charon tools. @@ -39,6 +102,9 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) + radas_config: Dict = data.get("radas", None) + if radas_config: + self.__radas_config__: RadasConfig = RadasConfig(radas_config) def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -67,6 +133,9 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable + def get_radas_config(self) -> RadasConfig: + return self.__radas_config__ + def get_config(cfgPath=None) -> CharonConfig: config_file_path = cfgPath diff --git a/charon/schemas/charon.json b/charon/schemas/charon.json index f6a931d1..3ffde818 100644 --- a/charon/schemas/charon.json +++ b/charon/schemas/charon.json @@ -30,6 +30,44 @@ "type": "string", "description": "signature command to be used for signature" }, + "radas": { + "type": "object", + "descrition": "", + "properties": { + "umb_host": { + "type": "string", + "description": "The host of UMB" + }, + "umb_host_port": { + "type": "string", + "description": "The port of UMB host" + }, + "result_queue": { + "type": "string", + "description": "The queue in UMB to receive radas signing result" + }, + "request_queue": { + "type": "string", + "description": "The queue in UMB to send signing request to RADAS" + }, + "client_ca": { + "type": "string", + "description": "the client ca file path" + }, + "client_key": { + "type": "string", + "description": "the client key file path" + }, + "client_key_pass_file":{ + "type": "string", + "description": "the file contains password of the client key" + }, + "root_ca": { + "type": "string", + "description": "the root ca file path" + } + } + }, "targets": { "type": "object", "patternProperties": { diff --git a/tests/test_config_radas.py b/tests/test_config_radas.py new file mode 100644 index 00000000..152dc1c2 --- /dev/null +++ b/tests/test_config_radas.py @@ -0,0 +1,226 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import unittest +import os +import charon.config as config +import shutil +import tempfile +from tests.base import BaseTest +from charon.utils.files import overwrite_file + + +class RadasConfigTest(unittest.TestCase): + def setUp(self) -> None: + self.__base = BaseTest() + self.__prepare_ca() + + def tearDown(self) -> None: + self.__base.tearDown() + self.__clear_ca() + + def test_full_radas_config(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + root_ca: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, + self.__client_key_pass_file, self.__root_ca) + print(radas_settings) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertTrue(rconf.validate()) + + def test_missing_umb_host(self): + radas_settings = """ +radas: + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_missing_result_queue(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_missing_request_queue(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_client_ca(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + os.remove(self.__client_ca_path) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_client_key(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + os.remove(self.__client_key_path) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_client_password_file(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + os.remove(self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_root_ca(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_queue: queue.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + root_ca: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, + self.__client_key_pass_file, self.__root_ca) + os.remove(self.__root_ca) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def __change_config_content(self, content: str): + self.__base.change_home() + config_base = self.__base.get_config_base() + os.mkdir(config_base) + self.__base.prepare_config(config_base, content) + + def __prepare_ca(self): + self.__tempdir = tempfile.mkdtemp() + self.__client_ca_path = os.path.join(self.__tempdir, "client_ca.crt") + self.__client_key_path = os.path.join(self.__tempdir, "client_key.crt") + self.__client_key_pass_file = os.path.join(self.__tempdir, "client_key_password.txt") + self.__root_ca = os.path.join(self.__tempdir, "root_ca.crt") + overwrite_file(self.__client_ca_path, "client ca") + overwrite_file(self.__client_key_path, "client key") + overwrite_file(self.__client_key_pass_file, "it's password") + overwrite_file(self.__root_ca, "root ca") + + def __clear_ca(self): + shutil.rmtree(self.__tempdir) From 2a2bef039d161b49556cdc5f627d798805f8c21e Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 28 Apr 2025 21:47:11 +0800 Subject: [PATCH 02/37] RADAS: Add sign command skeleton --- charon/cmd/__init__.py | 4 ++ charon/cmd/cmd_sign.py | 148 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 charon/cmd/cmd_sign.py diff --git a/charon/cmd/__init__.py b/charon/cmd/__init__.py index e2f54677..985d7f79 100644 --- a/charon/cmd/__init__.py +++ b/charon/cmd/__init__.py @@ -19,6 +19,7 @@ from charon.cmd.cmd_index import index from charon.cmd.cmd_checksum import init_checksum, checksum from charon.cmd.cmd_cache import init_cf, cf +from charon.cmd.cmd_sign import sign @group() @@ -43,3 +44,6 @@ def cli(ctx): # init checksum command init_checksum() cli.add_command(checksum) + +# radas sign cmd +cli.add_command(sign) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py new file mode 100644 index 00000000..9578183e --- /dev/null +++ b/charon/cmd/cmd_sign.py @@ -0,0 +1,148 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import List + +from charon.config import get_config, RadasConfig +from charon.cmd.internal import ( + _decide_mode, _safe_delete +) +from click import command, option, argument + +import traceback +import logging +import sys +import datetime + +logger = logging.getLogger(__name__) + + +@argument( + "repo_url", + type=str +) +@option( + "--requester", + "-r", + help=""" + The requester who sends the signing request. + """ +) +@option( + "--result_path", + "-p", + help=""" + The path which will save the sign result file. + """ +) +@option( + "--ignore_patterns", + "-i", + multiple=True, + help=""" + The regex patterns list to filter out the files which should + not be allowed to upload to S3. Can accept more than one pattern. + """ +) +@option( + "--work_dir", + "-w", + help=""" + The temporary working directory into which archives should + be extracted, when needed. + """ +) +@option( + "--config", + "-c", + help=""" + The charon configuration yaml file path. Default is + $HOME/.charon/charon.yaml + """ +) +@option( + "--sign_key", + "-k", + help=""" + rpm-sign key to be used, will replace {{ key }} in default configuration for signature. + Does noting if detach_signature_command does not contain {{ key }} field. + """ +) +@option( + "--debug", + "-D", + help="Debug mode, will print all debug logs for problem tracking.", + is_flag=True, + default=False +) +@option( + "--quiet", + "-q", + help="Quiet mode, will shrink most of the logs except warning and errors.", + is_flag=True, + default=False +) +@command() +def sign( + repo_url: str, + requester: str, + result_path: str, + ignore_patterns: List[str] = None, + work_dir: str = None, + config: str = None, + sign_key: str = "redhatdevel", + debug=False, + quiet=False, + dryrun=False +): + """Do signing against files in the repo zip in repo_url through + radas service. The repo_url points to the maven zip repository + in quay.io, which will be sent as the source of the signing. + """ + tmp_dir = work_dir + logger.debug("%s", ignore_patterns) + try: + current = datetime.datetime.now().strftime("%Y%m%d%I%M") + _decide_mode("radas_sign", current, is_quiet=quiet, is_debug=debug) + if dryrun: + logger.info("Running in dry-run mode, no files will signed.") + conf = get_config(config) + if not conf: + logger.error("The charon configuration is not valid!") + sys.exit(1) + radas_conf = conf.get_radas_config() + if not radas_conf or not radas_conf.validate(): + logger.error("The configuration for radas is not valid!") + sys.exit(1) + sign_in_radas(repo_url, requester, sign_key, result_path, radas_conf) + except Exception: + print(traceback.format_exc()) + sys.exit(2) # distinguish between exception and bad config or bad state + finally: + if not debug and tmp_dir: + _safe_delete(tmp_dir) + + +def sign_in_radas(repo_url: str, + requester: str, + sign_key: str, + result_path: str, + radas_config: RadasConfig): + '''This function will be responsible to do the overall controlling of the whole process, + like trigger the send and register the receiver, and control the wait and timeout there. + ''' + logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s," + "radas_config: %s", repo_url, requester, sign_key, result_path, radas_config) + logger.info("Not implemented yet!") From 5c7f7f4dc8a570b8a51c13b4a2a68421c263ee0a Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 13 May 2025 11:48:00 +0800 Subject: [PATCH 03/37] Feat: Support to accept the response of signing result from RADAS --- charon/constants.py | 3 + charon/pkgs/maven.py | 35 ++++- charon/pkgs/oras_client.py | 64 ++++++++ charon/pkgs/radas_signature_handler.py | 210 +++++++++++++++++++++++++ requirements.txt | 2 + 5 files changed, 309 insertions(+), 5 deletions(-) create mode 100644 charon/pkgs/oras_client.py create mode 100644 charon/pkgs/radas_signature_handler.py diff --git a/charon/constants.py b/charon/constants.py index 6751aecd..4c8320db 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -175,3 +175,6 @@ DEFAULT_ERRORS_LOG = "errors.log" DEFAULT_REGISTRY = "localhost" +DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" +DEFAULT_RADAS_SIGN_TIMEOUT_COUNT = 10 +DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 \ No newline at end of file diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 9f50f35b..900cc4cb 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -16,6 +16,7 @@ from charon.utils.files import HashType import charon.pkgs.indexing as indexing import charon.pkgs.signature as signature +import charon.pkgs.radas_signature_handler as radas_signature from charon.utils.files import overwrite_file, digest, write_manifest from charon.utils.archive import extract_zip_all from charon.utils.strings import remove_prefix @@ -408,11 +409,35 @@ def handle_maven_uploading( if cf_enable: cf_invalidate_paths.extend(archetype_files) - # 10. Generate signature file if contain_signature is set to True - if gen_sign: - conf = get_config(config) - if not conf: - sys.exit(1) + # 10. Generate signature file if radas sign is enabled, or do detached sign if contain_signature is set to True + conf = get_config(config) + if not conf: + sys.exit(1) + + if conf.get_radas_sign_enabled(): + logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) + (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign(top_level) + if not _generated_signs: + logger.error( + "No sign result files were downloaded, " + "please make sure the sign process is already done and without timeout") + return (tmp_root, False) + + failed_metas.extend(_failed_metas) + generated_signs.extend(_generated_signs) + logger.info("Singature generation against radas done.\n") + + logger.info("Start upload radas singature files to s3 bucket %s\n", bucket_name) + _failed_metas = s3_client.upload_signatures( + meta_file_paths=generated_signs, + target=(bucket_name, prefix), + product=None, + root=top_level + ) + failed_metas.extend(_failed_metas) + logger.info("Signature files uploading against radas done.\n") + + elif gen_sign: suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf) command = conf.get_detach_signature_command() artifacts = [s for s in valid_mvn_paths if not s.endswith(tuple(suffix_list))] diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py new file mode 100644 index 00000000..fca62006 --- /dev/null +++ b/charon/pkgs/oras_client.py @@ -0,0 +1,64 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oras.client +import logging +from charon.config import get_config +from typing import List + +logger = logging.getLogger(__name__) + +class OrasClient: + """ + Wrapper for oras‑py’s OrasClient, deciding whether to login based on config. + """ + + def __init__(self): + self.conf = get_config() + self.client = oras.client.OrasClient() + + def login_if_needed(self) -> None: + """ + If quay_radas_auth_enabled is true, call login to authenticate. + """ + + if self.conf and self.conf.is_quay_radas_auth_enabled(): + logger.info("Logging in to registry.") + res = self.client.login( + hostname=self.conf.get_quay_radas_registry(), + username=self.conf.get_quay_radas_username(), + password=self.conf.get_quay_radas_password(), + ) + logger.info(res) + else: + logger.info("Registry auth not enabled, skip login.") + + def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: + """ + Call oras‑py’s pull method to pull the remote file to local. + Args: + result_reference_url (str): Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). + sign_result_loc (str): Local save path (e.g. “/tmp/sign”). + """ + files = [] + try: + self.login_if_needed() + # the filename should be possibly named by the digest hash value based on the oras source code + files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) + logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) + except Exception as e: + logger.error("Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e) + finally: + return files \ No newline at end of file diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py new file mode 100644 index 00000000..5bacdab7 --- /dev/null +++ b/charon/pkgs/radas_signature_handler.py @@ -0,0 +1,210 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import proton +import proton.handlers +import threading +import logging +import json +import os +import asyncio +from typing import List, Any, Tuple, Callable, Dict +from charon.config import get_config +from charon.constants import DEFAULT_SIGN_RESULT_LOC +from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_COUNT +from charon.constants import DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC +from charon.pkgs.oras_client import OrasClient + +logger = logging.getLogger(__name__) + +class SignHandler: + """ + Handle the sign result status management + """ + _is_processing: bool = True + _downloaded_files: List[str] = [] + + @classmethod + def is_processing(cls) -> bool: + return cls._is_processing + + @classmethod + def get_downloaded_files(cls) -> List[str]: + return cls._downloaded_files.copy() + + @classmethod + def set_processing(cls, value: bool) -> None: + cls._is_processing = value + + @classmethod + def set_downloaded_files(cls, files: List[str]) -> None: + cls._downloaded_files = files + +class UmbListener(proton.handlers.MessagingHandler): + """ + UmbListener class (AMQP version), register this when setup UmbClient + """ + + def __init__(self) -> None: + super().__init__() + + def on_start(self, event: proton.Event) -> None: + """ + On start callback + """ + conf = get_config() + if not conf: + sys.exit(1) + event.container.create_receiver(conf.get_amqp_queue()) + + def on_message(self, event: proton.Event) -> None: + """ + On message callback + """ + # handle response from radas in a thread + thread = threading.Thread( + target=self._process_message, + args=[event.message.body] + ) + thread.start() + + def on_error(self, event: proton.Event) -> None: + """ + On error callback + """ + logger.error("Received an error event:\n%s", event) + + def on_disconnected(self, event: proton.Event) -> None: + """ + On disconnected callback + """ + logger.error("Disconnected from AMQP broker.") + + def _process_message(msg: Any) -> None: + """ + Process a message received from UMB + Args: + msg: The message body received + """ + try: + msg_dict = json.loads(msg) + result_reference_url = msg_dict.get("result_reference") + + if not result_reference_url: + logger.warning("Not found result_reference in message，ignore.") + return + + conf = get_config() + if not conf: + sign_result_loc = DEFAULT_SIGN_RESULT_LOC + sign_result_loc = os.getenv("SIGN_RESULT_LOC") or conf.get_sign_result_loc() + logger.info("Using SIGN RESULT LOC: %s", sign_result_loc) + + sign_result_parent_dir = os.path.dirname(sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) + + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, + sign_result_loc=sign_result_loc + ) + SignHandler.set_downloaded_files(files) + finally: + SignHandler.set_processing(False) + +def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: + """ + Generate .asc files based on RADAS sign result json file + """ + conf = get_config() + timeout_count = conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT + wait_interval_sec = conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + wait_count = 0 + while SignHandler.is_processing(): + wait_count += 1 + if wait_count > timeout_count: + logger.warning("Timeout when waiting for sign response.") + break + time.sleep(wait_interval_sec) + + files = SignHandler.get_downloaded_files() + if not files: + return [], [] + + # should only have the single sign result json file from the radas registry + json_file_path = files[0] + try: + with open(json_file_path, 'r') as f: + data = json.load(f) + except Exception as e: + logger.error(f"Failed to read or parse the JSON file: {e}") + raise + + async def generate_single_sign_file( + file_path: str, signature: str, failed_paths: List[str], generated_signs: List[str], + sem: asyncio.BoundedSemaphore + ): + async with sem: + if not file_path or not signature: + logger.error(f"Invalid JSON entry") + return + # remove the root path maven-repository + filename = file_path.split("/", 1)[1] + signature = item.get("signature") + + artifact_path = os.path.join(top_level, filename) + asc_filename = f"{filename}.asc" + signature_path = os.path.join(top_level, asc_filename) + + if not os.path.isfile(artifact_path): + logger.warning("Artifact missing, skip signature file generation") + return + + try: + with open(signature_path, 'w') as asc_file: + asc_file.write(signature) + generated_signs.append(signature_path) + logger.info(f"Generated .asc file: {signature_path}") + except Exception as e: + failed_paths.append(signature_path) + logger.error(f"Failed to write .asc file for {artifact_path}: {e}") + + result = data.get("result", []) + return __do_path_cut_and( + path_handler=generate_single_sign_file, + data=result + ) + +def __do_path_cut_and( + path_handler: Callable, + data: List[Dict[str, str]] +) -> Tuple[List[str], List[str]]: + + failed_paths: List[str] = [] + generated_signs: List[str] = [] + tasks = [] + sem = asyncio.BoundedSemaphore(10) + for item in data: + file_path = item.get("file") + signature = item.get("signature") + tasks.append( + asyncio.ensure_future( + path_handler(file_path, signature, failed_paths, generated_signs, sem) + ) + ) + + loop = asyncio.get_event_loop() + loop.run_until_complete(asyncio.gather(*tasks)) + return (failed_paths, generated_signs) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7919fc27..75bb4b60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,5 @@ subresource-integrity>=0.2 jsonschema>=4.9.1 urllib3>=1.25.10 semantic-version>=2.10.0 +oras>=0.2.31 +python-qpid-proton>=0.39.0 \ No newline at end of file From 070c45ad3fd1a7fc95e745eef1eb3bd4092701a7 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 13 May 2025 13:31:44 +0800 Subject: [PATCH 04/37] Format Code and fix typo --- charon/config.py | 6 +-- charon/constants.py | 2 +- charon/pkgs/maven.py | 5 ++- charon/pkgs/oras_client.py | 16 ++++--- charon/pkgs/radas_signature_handler.py | 58 ++++++++++++++------------ 5 files changed, 48 insertions(+), 39 deletions(-) diff --git a/charon/config.py b/charon/config.py index 35efe6eb..c9cbdc59 100644 --- a/charon/config.py +++ b/charon/config.py @@ -141,14 +141,12 @@ def get_config(cfgPath=None) -> CharonConfig: config_file_path = cfgPath if not config_file_path or not os.path.isfile(config_file_path): config_file_path = os.path.join(os.getenv("HOME", ""), ".charon", CONFIG_FILE) - data = read_yaml_from_file_path(config_file_path, 'schemas/charon.json') + data = read_yaml_from_file_path(config_file_path, "schemas/charon.json") return CharonConfig(data) def get_template(template_file: str) -> str: - template = os.path.join( - os.getenv("HOME", ''), ".charon/template", template_file - ) + template = os.path.join(os.getenv("HOME", ""), ".charon/template", template_file) if os.path.isfile(template): with open(template, encoding="utf-8") as file_: return file_.read() diff --git a/charon/constants.py b/charon/constants.py index 4c8320db..c8b8d125 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -177,4 +177,4 @@ DEFAULT_REGISTRY = "localhost" DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" DEFAULT_RADAS_SIGN_TIMEOUT_COUNT = 10 -DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 \ No newline at end of file +DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 900cc4cb..8c14d822 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -409,12 +409,13 @@ def handle_maven_uploading( if cf_enable: cf_invalidate_paths.extend(archetype_files) - # 10. Generate signature file if radas sign is enabled, or do detached sign if contain_signature is set to True + # 10. Generate signature file if radas sign is enabled, + # or do detached sign if contain_signature is set to True conf = get_config(config) if not conf: sys.exit(1) - if conf.get_radas_sign_enabled(): + if conf.is_radas_sign_enabled(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign(top_level) if not _generated_signs: diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py index fca62006..95310215 100644 --- a/charon/pkgs/oras_client.py +++ b/charon/pkgs/oras_client.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import oras.client import logging from charon.config import get_config @@ -20,6 +21,7 @@ logger = logging.getLogger(__name__) + class OrasClient: """ Wrapper for oras‑py’s OrasClient, deciding whether to login based on config. @@ -49,16 +51,18 @@ def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: """ Call oras‑py’s pull method to pull the remote file to local. Args: - result_reference_url (str): Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). - sign_result_loc (str): Local save path (e.g. “/tmp/sign”). + result_reference_url (str): + Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). + sign_result_loc (str): + Local save path (e.g. “/tmp/sign”). """ files = [] try: self.login_if_needed() - # the filename should be possibly named by the digest hash value based on the oras source code files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) except Exception as e: - logger.error("Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e) - finally: - return files \ No newline at end of file + logger.error( + "Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e + ) + return files diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 5bacdab7..9120bc9b 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import proton import proton.handlers import threading @@ -20,6 +21,8 @@ import json import os import asyncio +import sys +import time from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config from charon.constants import DEFAULT_SIGN_RESULT_LOC @@ -29,10 +32,12 @@ logger = logging.getLogger(__name__) + class SignHandler: """ Handle the sign result status management """ + _is_processing: bool = True _downloaded_files: List[str] = [] @@ -52,6 +57,7 @@ def set_processing(cls, value: bool) -> None: def set_downloaded_files(cls, files: List[str]) -> None: cls._downloaded_files = files + class UmbListener(proton.handlers.MessagingHandler): """ UmbListener class (AMQP version), register this when setup UmbClient @@ -74,10 +80,7 @@ def on_message(self, event: proton.Event) -> None: On message callback """ # handle response from radas in a thread - thread = threading.Thread( - target=self._process_message, - args=[event.message.body] - ) + thread = threading.Thread(target=self._process_message, args=[event.message.body]) thread.start() def on_error(self, event: proton.Event) -> None: @@ -103,8 +106,8 @@ def _process_message(msg: Any) -> None: result_reference_url = msg_dict.get("result_reference") if not result_reference_url: - logger.warning("Not found result_reference in message，ignore.") - return + logger.warning("Not found result_reference in message，ignore.") + return conf = get_config() if not conf: @@ -117,20 +120,24 @@ def _process_message(msg: Any) -> None: oras_client = OrasClient() files = oras_client.pull( - result_reference_url=result_reference_url, - sign_result_loc=sign_result_loc + result_reference_url=result_reference_url, sign_result_loc=sign_result_loc ) SignHandler.set_downloaded_files(files) finally: SignHandler.set_processing(False) + def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file """ conf = get_config() - timeout_count = conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT - wait_interval_sec = conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + timeout_count = ( + conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT + ) + wait_interval_sec = ( + conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + ) wait_count = 0 while SignHandler.is_processing(): wait_count += 1 @@ -146,23 +153,25 @@ def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: # should only have the single sign result json file from the radas registry json_file_path = files[0] try: - with open(json_file_path, 'r') as f: + with open(json_file_path, "r") as f: data = json.load(f) except Exception as e: - logger.error(f"Failed to read or parse the JSON file: {e}") + logger.error("Failed to read or parse the JSON file: %s", e) raise async def generate_single_sign_file( - file_path: str, signature: str, failed_paths: List[str], generated_signs: List[str], - sem: asyncio.BoundedSemaphore + file_path: str, + signature: str, + failed_paths: List[str], + generated_signs: List[str], + sem: asyncio.BoundedSemaphore, ): async with sem: if not file_path or not signature: - logger.error(f"Invalid JSON entry") + logger.error("Invalid JSON entry") return # remove the root path maven-repository filename = file_path.split("/", 1)[1] - signature = item.get("signature") artifact_path = os.path.join(top_level, filename) asc_filename = f"{filename}.asc" @@ -173,23 +182,20 @@ async def generate_single_sign_file( return try: - with open(signature_path, 'w') as asc_file: + with open(signature_path, "w") as asc_file: asc_file.write(signature) generated_signs.append(signature_path) - logger.info(f"Generated .asc file: {signature_path}") + logger.info("Generated .asc file: %s", signature_path) except Exception as e: failed_paths.append(signature_path) - logger.error(f"Failed to write .asc file for {artifact_path}: {e}") + logger.error("Failed to write .asc file for %s: %s", artifact_path, e) result = data.get("result", []) - return __do_path_cut_and( - path_handler=generate_single_sign_file, - data=result - ) + return __do_path_cut_and(path_handler=generate_single_sign_file, data=result) + def __do_path_cut_and( - path_handler: Callable, - data: List[Dict[str, str]] + path_handler: Callable, data: List[Dict[str, str]] ) -> Tuple[List[str], List[str]]: failed_paths: List[str] = [] @@ -207,4 +213,4 @@ def __do_path_cut_and( loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather(*tasks)) - return (failed_paths, generated_signs) \ No newline at end of file + return (failed_paths, generated_signs) From 558d09cacb2babb9314c2f33fbc71f29a6a2b9f4 Mon Sep 17 00:00:00 2001 From: yma Date: Mon, 19 May 2025 15:12:26 +0800 Subject: [PATCH 05/37] Rename radas_sign_timeout_retry_count and radas_sign_timeout_retry_interval --- charon/config.py | 15 +++++++++++++-- charon/constants.py | 4 ++-- charon/pkgs/radas_signature_handler.py | 19 +++++++++++-------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/charon/config.py b/charon/config.py index c9cbdc59..b6eb07cd 100644 --- a/charon/config.py +++ b/charon/config.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import logging import os from typing import Dict, List, Optional @@ -34,6 +35,10 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) + self.__radas_sign_timeout_retry_interval: int = data.get( + "radas_sign_timeout_retry_interval", 60 + ) def validate(self) -> bool: if not self.__umb_host: @@ -60,7 +65,7 @@ def validate(self) -> bool: return True def umb_target(self) -> str: - return f'amqps://{self.__umb_host}:{self.__umb_host_port}' + return f"amqps://{self.__umb_host}:{self.__umb_host_port}" def result_queue(self) -> str: return self.__result_queue @@ -77,7 +82,7 @@ def client_key(self) -> str: def client_key_password(self) -> str: pass_file = self.__client_key_pass_file if os.access(pass_file, os.R_OK): - with open(pass_file, 'r') as f: + with open(pass_file, "r") as f: return f.read() elif pass_file: logger.warning("The key password file is not accessible. Will ignore the password.") @@ -86,6 +91,12 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca + def radas_sign_timeout_retry_count(self) -> int: + return self.__radas_sign_timeout_retry_count + + def radas_sign_timeout_retry_interval(self) -> int: + return self.__radas_sign_timeout_retry_interval + class CharonConfig(object): """CharonConfig is used to store all configurations for charon diff --git a/charon/constants.py b/charon/constants.py index c8b8d125..25970c36 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -176,5 +176,5 @@ DEFAULT_REGISTRY = "localhost" DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" -DEFAULT_RADAS_SIGN_TIMEOUT_COUNT = 10 -DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 9120bc9b..eb755086 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -26,8 +26,8 @@ from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config from charon.constants import DEFAULT_SIGN_RESULT_LOC -from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_COUNT -from charon.constants import DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC +from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT +from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL from charon.pkgs.oras_client import OrasClient logger = logging.getLogger(__name__) @@ -132,19 +132,22 @@ def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: Generate .asc files based on RADAS sign result json file """ conf = get_config() - timeout_count = ( - conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT + rconf = conf.get_radas_config() if conf else None + timeout_retry_count = ( + rconf.radas_sign_timeout_retry_count() if rconf else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT ) - wait_interval_sec = ( - conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + timeout_retry_interval = ( + rconf.radas_sign_timeout_retry_interval() + if conf + else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL ) wait_count = 0 while SignHandler.is_processing(): wait_count += 1 - if wait_count > timeout_count: + if wait_count > timeout_retry_count: logger.warning("Timeout when waiting for sign response.") break - time.sleep(wait_interval_sec) + time.sleep(timeout_retry_interval) files = SignHandler.get_downloaded_files() if not files: From dbbdf5eb04fe3c339543be61be32fbcecf711219 Mon Sep 17 00:00:00 2001 From: yma Date: Mon, 19 May 2025 21:00:36 +0800 Subject: [PATCH 06/37] Use oras registry_config and registry url parse to finalize login --- charon/config.py | 11 +++++++++++ charon/pkgs/oras_client.py | 22 +++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/charon/config.py b/charon/config.py index b6eb07cd..fe3c1174 100644 --- a/charon/config.py +++ b/charon/config.py @@ -35,6 +35,9 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + self.__quay_radas_registry_config: str = data.get( + "quay_radas_registry_config", os.path.join(os.getenv("HOME", ""), ".oras/config.json") + ) self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 @@ -62,6 +65,11 @@ def validate(self) -> bool: if self.__root_ca and not os.access(self.__root_ca, os.R_OK): logger.error("The root ca file is not valid!") return False + if self.__quay_radas_registry_config and not os.access( + self.__quay_radas_registry_config, os.R_OK + ): + logger.error("The quay registry config for oras is not valid!") + return False return True def umb_target(self) -> str: @@ -91,6 +99,9 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca + def quay_radas_registry_config(self) -> str: + return self.__quay_radas_registry_config + def radas_sign_timeout_retry_count(self) -> int: return self.__radas_sign_timeout_retry_count diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py index 95310215..b5446def 100644 --- a/charon/pkgs/oras_client.py +++ b/charon/pkgs/oras_client.py @@ -18,6 +18,7 @@ import logging from charon.config import get_config from typing import List +from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -31,21 +32,24 @@ def __init__(self): self.conf = get_config() self.client = oras.client.OrasClient() - def login_if_needed(self) -> None: + def login_if_needed(self, registry: str) -> None: """ - If quay_radas_auth_enabled is true, call login to authenticate. + If quay_radas_registry_config is provided, call login to authenticate. """ + if not registry.startswith("http://") and not registry.startswith("https://"): + registry = "https://" + registry + registry = urlparse(registry).netloc - if self.conf and self.conf.is_quay_radas_auth_enabled(): - logger.info("Logging in to registry.") + rconf = self.conf.get_radas_config() if self.conf else None + if rconf and rconf.quay_radas_registry_config(): + logger.info("Logging in to registry: %s", registry) res = self.client.login( - hostname=self.conf.get_quay_radas_registry(), - username=self.conf.get_quay_radas_username(), - password=self.conf.get_quay_radas_password(), + hostname=registry, + config_path=rconf.quay_radas_registry_config(), ) logger.info(res) else: - logger.info("Registry auth not enabled, skip login.") + logger.info("Registry config is not provided, skip login.") def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: """ @@ -58,7 +62,7 @@ def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: """ files = [] try: - self.login_if_needed() + self.login_if_needed(registry=result_reference_url) files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) except Exception as e: From c928769660470701305243d60f12867a7946ca2c Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 09:32:24 +0800 Subject: [PATCH 07/37] Change sign_result_loc to cmd flag both for sign request and maven upload --- charon/cmd/cmd_upload.py | 16 +++- charon/config.py | 7 +- charon/constants.py | 1 - charon/pkgs/maven.py | 15 ++-- charon/pkgs/radas_signature_handler.py | 112 +++++++++++-------------- 5 files changed, 78 insertions(+), 73 deletions(-) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index a867df01..d56a644d 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -136,6 +136,16 @@ default=False ) @option("--dryrun", "-n", is_flag=True, default=False) +@option( + "--sign_result_loc", + "-l", + default="/tmp/sign", + help=""" + The local save path for oras to pull the radas signature result. + Sign request will use this path to download the signature result, + Upload will use the file on this path to generate the corresponding .asc files + """, +) @command() def upload( repo: str, @@ -150,7 +160,8 @@ def upload( sign_key: str = "redhatdevel", debug=False, quiet=False, - dryrun=False + dryrun=False, + sign_result_loc="/tmp/sign" ): """Upload all files from a released product REPO to Ronda Service. The REPO points to a product released tarball which @@ -221,7 +232,8 @@ def upload( key=sign_key, dry_run=dryrun, manifest_bucket_name=manifest_bucket_name, - config=config + config=config, + sign_result_loc=sign_result_loc ) if not succeeded: sys.exit(1) diff --git a/charon/config.py b/charon/config.py index fe3c1174..ce418b33 100644 --- a/charon/config.py +++ b/charon/config.py @@ -48,7 +48,7 @@ def validate(self) -> bool: logger.error("Missing host name setting for UMB!") return False if not self.__result_queue: - logger.error("Missing the queue setting to receive siging result in UMB!") + logger.error("Missing the queue setting to receive signing result in UMB!") return False if not self.__request_queue: logger.error("Missing the queue setting to send signing request in UMB!") @@ -124,8 +124,10 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) + self.__radas_config_enable: bool = data.get("radas_config_enable", False) radas_config: Dict = data.get("radas", None) if radas_config: + self.__radas_config_enable = True self.__radas_config__: RadasConfig = RadasConfig(radas_config) def get_ignore_patterns(self) -> List[str]: @@ -155,6 +157,9 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable + def is_radas_config_enable(self) -> bool: + return self.__radas_config_enable + def get_radas_config(self) -> RadasConfig: return self.__radas_config__ diff --git a/charon/constants.py b/charon/constants.py index 25970c36..35ea560a 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -175,6 +175,5 @@ DEFAULT_ERRORS_LOG = "errors.log" DEFAULT_REGISTRY = "localhost" -DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 8c14d822..dde1a4cb 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -275,7 +275,8 @@ def handle_maven_uploading( key=None, dry_run=False, manifest_bucket_name=None, - config=None + config=None, + sign_result_loc="/tmp/sign" ) -> Tuple[str, bool]: """ Handle the maven product release tarball uploading process. * repo is the location of the tarball in filesystem @@ -415,9 +416,11 @@ def handle_maven_uploading( if not conf: sys.exit(1) - if conf.is_radas_sign_enabled(): + if conf.is_radas_config_enable(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) - (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign(top_level) + (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( + top_level=top_level, sign_result_loc=sign_result_loc + ) if not _generated_signs: logger.error( "No sign result files were downloaded, " @@ -426,9 +429,9 @@ def handle_maven_uploading( failed_metas.extend(_failed_metas) generated_signs.extend(_generated_signs) - logger.info("Singature generation against radas done.\n") + logger.info("Radas signature files generation done.\n") - logger.info("Start upload radas singature files to s3 bucket %s\n", bucket_name) + logger.info("Start upload radas signature files to s3 bucket %s\n", bucket_name) _failed_metas = s3_client.upload_signatures( meta_file_paths=generated_signs, target=(bucket_name, prefix), @@ -436,7 +439,7 @@ def handle_maven_uploading( root=top_level ) failed_metas.extend(_failed_metas) - logger.info("Signature files uploading against radas done.\n") + logger.info("Radas signature files uploading done.\n") elif gen_sign: suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index eb755086..9ef00d8f 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -14,8 +14,6 @@ limitations under the License. """ -import proton -import proton.handlers import threading import logging import json @@ -25,57 +23,40 @@ import time from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config -from charon.constants import DEFAULT_SIGN_RESULT_LOC from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL from charon.pkgs.oras_client import OrasClient +from proton import Event +from proton.handlers import MessagingHandler logger = logging.getLogger(__name__) -class SignHandler: - """ - Handle the sign result status management - """ - - _is_processing: bool = True - _downloaded_files: List[str] = [] - - @classmethod - def is_processing(cls) -> bool: - return cls._is_processing - - @classmethod - def get_downloaded_files(cls) -> List[str]: - return cls._downloaded_files.copy() - - @classmethod - def set_processing(cls, value: bool) -> None: - cls._is_processing = value - - @classmethod - def set_downloaded_files(cls, files: List[str]) -> None: - cls._downloaded_files = files - - -class UmbListener(proton.handlers.MessagingHandler): +class UmbListener(MessagingHandler): """ UmbListener class (AMQP version), register this when setup UmbClient + Attributes: + sign_result_loc (str): Local save path (e.g. “/tmp/sign”) for oras pull result, + this value transfers from the cmd flag, should register UmbListener when the client starts """ - def __init__(self) -> None: + def __init__(self, sign_result_loc: str) -> None: super().__init__() + self.sign_result_loc = sign_result_loc - def on_start(self, event: proton.Event) -> None: + def on_start(self, event: Event) -> None: """ On start callback """ conf = get_config() - if not conf: + rconf = conf.get_radas_config() if conf else None + if not rconf: sys.exit(1) - event.container.create_receiver(conf.get_amqp_queue()) + conn = event.container.connect(rconf.umb_target()) + event.container.create_receiver(conn, rconf.result_queue()) + logger.info("Listening on %s, queue: %s", rconf.umb_target(), rconf.result_queue()) - def on_message(self, event: proton.Event) -> None: + def on_message(self, event: Event) -> None: """ On message callback """ @@ -83,51 +64,43 @@ def on_message(self, event: proton.Event) -> None: thread = threading.Thread(target=self._process_message, args=[event.message.body]) thread.start() - def on_error(self, event: proton.Event) -> None: + def on_connection_error(self, event: Event) -> None: """ - On error callback + On connection error callback """ logger.error("Received an error event:\n%s", event) - def on_disconnected(self, event: proton.Event) -> None: + def on_disconnected(self, event: Event) -> None: """ On disconnected callback """ logger.error("Disconnected from AMQP broker.") - def _process_message(msg: Any) -> None: + def _process_message(self, msg: Any) -> None: """ Process a message received from UMB Args: msg: The message body received """ - try: - msg_dict = json.loads(msg) - result_reference_url = msg_dict.get("result_reference") + msg_dict = json.loads(msg) + result_reference_url = msg_dict.get("result_reference") - if not result_reference_url: - logger.warning("Not found result_reference in message，ignore.") - return - - conf = get_config() - if not conf: - sign_result_loc = DEFAULT_SIGN_RESULT_LOC - sign_result_loc = os.getenv("SIGN_RESULT_LOC") or conf.get_sign_result_loc() - logger.info("Using SIGN RESULT LOC: %s", sign_result_loc) + if not result_reference_url: + logger.warning("Not found result_reference in message，ignore.") + return - sign_result_parent_dir = os.path.dirname(sign_result_loc) - os.makedirs(sign_result_parent_dir, exist_ok=True) + logger.info("Using SIGN RESULT LOC: %s", self.sign_result_loc) + sign_result_parent_dir = os.path.dirname(self.sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) - oras_client = OrasClient() - files = oras_client.pull( - result_reference_url=result_reference_url, sign_result_loc=sign_result_loc - ) - SignHandler.set_downloaded_files(files) - finally: - SignHandler.set_processing(False) + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc + ) + logger.info("Number of files pulled: %d, path: %s", len(files), files[0]) -def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: +def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file """ @@ -138,21 +111,34 @@ def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: ) timeout_retry_interval = ( rconf.radas_sign_timeout_retry_interval() - if conf + if rconf else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL ) wait_count = 0 - while SignHandler.is_processing(): + + # Wait until files appear in the sign_result_loc directory + while True: + files = [ + os.path.join(sign_result_loc, f) + for f in os.listdir(sign_result_loc) + if os.path.isfile(os.path.join(sign_result_loc, f)) + ] + if files: # If files exist, break the loop + break + wait_count += 1 if wait_count > timeout_retry_count: logger.warning("Timeout when waiting for sign response.") break time.sleep(timeout_retry_interval) - files = SignHandler.get_downloaded_files() if not files: return [], [] + if len(files) > 1: + logger.error("Multiple files found in %s. Expected only one file.", sign_result_loc) + return [], [] + # should only have the single sign result json file from the radas registry json_file_path = files[0] try: From 01fa98842bfa971b6aa83b891bb57ced9b33bd1f Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 10:22:18 +0800 Subject: [PATCH 08/37] Update quay_radas_registry_config to default None since it's not necessary for public --- charon/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/charon/config.py b/charon/config.py index ce418b33..ca238e63 100644 --- a/charon/config.py +++ b/charon/config.py @@ -35,9 +35,7 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") - self.__quay_radas_registry_config: str = data.get( - "quay_radas_registry_config", os.path.join(os.getenv("HOME", ""), ".oras/config.json") - ) + self.__quay_radas_registry_config: str = data.get("quay_radas_registry_config", None) self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 From ea019f73f2c0ee7796389e87809b92a95c985b26 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 11:12:06 +0800 Subject: [PATCH 09/37] Use radas_config validate instead of is_radas_config_enable option --- charon/config.py | 10 +++------- charon/pkgs/maven.py | 3 ++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/charon/config.py b/charon/config.py index ca238e63..a53bdaa6 100644 --- a/charon/config.py +++ b/charon/config.py @@ -122,11 +122,10 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) - self.__radas_config_enable: bool = data.get("radas_config_enable", False) + self.__radas_config__: Optional[RadasConfig] = None radas_config: Dict = data.get("radas", None) if radas_config: - self.__radas_config_enable = True - self.__radas_config__: RadasConfig = RadasConfig(radas_config) + self.__radas_config__ = RadasConfig(radas_config) def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -155,10 +154,7 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable - def is_radas_config_enable(self) -> bool: - return self.__radas_config_enable - - def get_radas_config(self) -> RadasConfig: + def get_radas_config(self) -> Optional[RadasConfig]: return self.__radas_config__ diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index dde1a4cb..3971fd06 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -416,7 +416,8 @@ def handle_maven_uploading( if not conf: sys.exit(1) - if conf.is_radas_config_enable(): + rconf = conf.get_radas_config() + if rconf and rconf.validate(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( top_level=top_level, sign_result_loc=sign_result_loc From f060c5ff625bd88400d18aa0a3cd6fe992418f1c Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 14:12:41 +0800 Subject: [PATCH 10/37] Ignore the registry config if the provided config path is not valid to read --- charon/config.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/charon/config.py b/charon/config.py index a53bdaa6..5e72f270 100644 --- a/charon/config.py +++ b/charon/config.py @@ -35,7 +35,9 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") - self.__quay_radas_registry_config: str = data.get("quay_radas_registry_config", None) + self.__quay_radas_registry_config: Optional[str] = data.get( + "quay_radas_registry_config", None + ) self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 @@ -66,8 +68,10 @@ def validate(self) -> bool: if self.__quay_radas_registry_config and not os.access( self.__quay_radas_registry_config, os.R_OK ): - logger.error("The quay registry config for oras is not valid!") - return False + self.__quay_radas_registry_config = None + logger.warning( + "The quay registry config for oras is not valid, will ignore the registry config!" + ) return True def umb_target(self) -> str: @@ -97,7 +101,7 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca - def quay_radas_registry_config(self) -> str: + def quay_radas_registry_config(self) -> Optional[str]: return self.__quay_radas_registry_config def radas_sign_timeout_retry_count(self) -> int: From 4b344b0d6b82110885aaa80fbd4e829c1d8194fc Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 15:43:39 +0800 Subject: [PATCH 11/37] Add is_radas_enabled unified method to check radas enablement --- charon/config.py | 3 +++ charon/pkgs/maven.py | 3 +-- charon/pkgs/radas_signature_handler.py | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/charon/config.py b/charon/config.py index 5e72f270..2995ffdf 100644 --- a/charon/config.py +++ b/charon/config.py @@ -158,6 +158,9 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable + def is_radas_enabled(self) -> bool: + return bool(self.__radas_config__ and self.__radas_config__.validate()) + def get_radas_config(self) -> Optional[RadasConfig]: return self.__radas_config__ diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 3971fd06..4ca1be0d 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -416,8 +416,7 @@ def handle_maven_uploading( if not conf: sys.exit(1) - rconf = conf.get_radas_config() - if rconf and rconf.validate(): + if conf.is_radas_enabled(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( top_level=top_level, sign_result_loc=sign_result_loc diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 9ef00d8f..27a14548 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -49,8 +49,12 @@ def on_start(self, event: Event) -> None: On start callback """ conf = get_config() - rconf = conf.get_radas_config() if conf else None - if not rconf: + if not (conf and conf.is_radas_enabled()): + sys.exit(1) + + rconf = conf.get_radas_config() + # explicit check to pass the type checker + if rconf is None: sys.exit(1) conn = event.container.connect(rconf.umb_target()) event.container.create_receiver(conn, rconf.result_queue()) From ae24bcd647b26d619e0d3bbeca8031fc821a65cc Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 21 May 2025 15:20:27 +0800 Subject: [PATCH 12/37] Change on_message process method without using threads --- charon/pkgs/radas_signature_handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 27a14548..a62a0d96 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -14,7 +14,6 @@ limitations under the License. """ -import threading import logging import json import os @@ -64,9 +63,7 @@ def on_message(self, event: Event) -> None: """ On message callback """ - # handle response from radas in a thread - thread = threading.Thread(target=self._process_message, args=[event.message.body]) - thread.start() + self._process_message(event.message.body) def on_connection_error(self, event: Event) -> None: """ From 81488ad68375e7d593c90e4ede14141a43f4daa7 Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 21 May 2025 17:29:44 +0800 Subject: [PATCH 13/37] Remove timeout retry handling for sign result fetch from maven upload --- charon/pkgs/radas_signature_handler.py | 35 ++++---------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index a62a0d96..c04f0bbf 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -19,11 +19,8 @@ import os import asyncio import sys -import time from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config -from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT -from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL from charon.pkgs.oras_client import OrasClient from proton import Event from proton.handlers import MessagingHandler @@ -105,33 +102,11 @@ def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str] """ Generate .asc files based on RADAS sign result json file """ - conf = get_config() - rconf = conf.get_radas_config() if conf else None - timeout_retry_count = ( - rconf.radas_sign_timeout_retry_count() if rconf else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT - ) - timeout_retry_interval = ( - rconf.radas_sign_timeout_retry_interval() - if rconf - else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL - ) - wait_count = 0 - - # Wait until files appear in the sign_result_loc directory - while True: - files = [ - os.path.join(sign_result_loc, f) - for f in os.listdir(sign_result_loc) - if os.path.isfile(os.path.join(sign_result_loc, f)) - ] - if files: # If files exist, break the loop - break - - wait_count += 1 - if wait_count > timeout_retry_count: - logger.warning("Timeout when waiting for sign response.") - break - time.sleep(timeout_retry_interval) + files = [ + os.path.join(sign_result_loc, f) + for f in os.listdir(sign_result_loc) + if os.path.isfile(os.path.join(sign_result_loc, f)) + ] if not files: return [], [] From d433464cc99afea2cd9d229d8ea76fb757d84204 Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 21 May 2025 17:29:44 +0800 Subject: [PATCH 14/37] Some changes on sign cmd and config --- charon/cmd/cmd_sign.py | 32 +++++++++++++------------- charon/config.py | 12 ++++++---- charon/pkgs/radas_signature_handler.py | 16 ++++++++++--- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index 9578183e..629281cb 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -15,7 +15,8 @@ """ from typing import List -from charon.config import get_config, RadasConfig +from charon.config import get_config +from charon.pkgs.radas_signature_handler import sign_in_radas from charon.cmd.internal import ( _decide_mode, _safe_delete ) @@ -126,23 +127,22 @@ def sign( if not radas_conf or not radas_conf.validate(): logger.error("The configuration for radas is not valid!") sys.exit(1) - sign_in_radas(repo_url, requester, sign_key, result_path, radas_conf) + # All ignore files in global config should also be ignored in signing. + ig_patterns = conf.get_ignore_patterns() + if ignore_patterns: + ig_patterns.extend(ignore_patterns) + args = { + "repo_url": repo_url, + "requester": requester, + "sign_key": sign_key, + "result_path": result_path, + "ignore_patterns": ig_patterns, + "radas_config": radas_conf + } + sign_in_radas(**args) # type: ignore except Exception: print(traceback.format_exc()) - sys.exit(2) # distinguish between exception and bad config or bad state + sys.exit(2) finally: if not debug and tmp_dir: _safe_delete(tmp_dir) - - -def sign_in_radas(repo_url: str, - requester: str, - sign_key: str, - result_path: str, - radas_config: RadasConfig): - '''This function will be responsible to do the overall controlling of the whole process, - like trigger the send and register the receiver, and control the wait and timeout there. - ''' - logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s," - "radas_config: %s", repo_url, requester, sign_key, result_path, radas_config) - logger.info("Not implemented yet!") diff --git a/charon/config.py b/charon/config.py index 2995ffdf..44bc9c77 100644 --- a/charon/config.py +++ b/charon/config.py @@ -126,10 +126,12 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) - self.__radas_config__: Optional[RadasConfig] = None radas_config: Dict = data.get("radas", None) if radas_config: - self.__radas_config__ = RadasConfig(radas_config) + self.__radas_config = RadasConfig(radas_config) + self.__radas_enabled = bool(self.__radas_config and self.__radas_config.validate()) + else: + self.__radas_enabled = False def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -159,10 +161,10 @@ def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable def is_radas_enabled(self) -> bool: - return bool(self.__radas_config__ and self.__radas_config__.validate()) + return self.__radas_enabled - def get_radas_config(self) -> Optional[RadasConfig]: - return self.__radas_config__ + def get_radas_config(self) -> RadasConfig: + return self.__radas_config def get_config(cfgPath=None) -> CharonConfig: diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index c04f0bbf..46794f49 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -20,7 +20,7 @@ import asyncio import sys from typing import List, Any, Tuple, Callable, Dict -from charon.config import get_config +from charon.config import get_config, RadasConfig from charon.pkgs.oras_client import OrasClient from proton import Event from proton.handlers import MessagingHandler @@ -28,9 +28,10 @@ logger = logging.getLogger(__name__) -class UmbListener(MessagingHandler): +class RadasReceiver(MessagingHandler): """ - UmbListener class (AMQP version), register this when setup UmbClient + This receiver will listen to UMB message queue to receive signing message for + signing result. Attributes: sign_result_loc (str): Local save path (e.g. “/tmp/sign”) for oras pull result, this value transfers from the cmd flag, should register UmbListener when the client starts @@ -179,3 +180,12 @@ def __do_path_cut_and( loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather(*tasks)) return (failed_paths, generated_signs) + + +def sign_in_radas(repo_url: str, + requester: str, + sign_key: str, + result_path: str, + ignore_patterns: List[str], + radas_config: RadasConfig): + logger.info("Start signing for %s", repo_url) From 229e44fe466cf3cfab14a410ff79a770b6b2d1a4 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 26 May 2025 20:35:28 +0800 Subject: [PATCH 15/37] Fix: add back the radas_config type for mypy check --- charon/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charon/config.py b/charon/config.py index 44bc9c77..5e7734d0 100644 --- a/charon/config.py +++ b/charon/config.py @@ -127,6 +127,7 @@ def __init__(self, data: Dict): self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) radas_config: Dict = data.get("radas", None) + self.__radas_config: Optional[RadasConfig] = None if radas_config: self.__radas_config = RadasConfig(radas_config) self.__radas_enabled = bool(self.__radas_config and self.__radas_config.validate()) @@ -163,7 +164,7 @@ def is_aws_cf_enable(self) -> bool: def is_radas_enabled(self) -> bool: return self.__radas_enabled - def get_radas_config(self) -> RadasConfig: + def get_radas_config(self) -> Optional[RadasConfig]: return self.__radas_config From 53b0196c2a1357105fc347bb6e36208e09f56190 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 27 May 2025 10:46:43 +0800 Subject: [PATCH 16/37] Add Unit tests for RADAS signing results parse and generation --- charon/pkgs/maven.py | 2 +- charon/pkgs/radas_signature_handler.py | 4 + tests/test_radas_sign_handler.py | 162 +++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 tests/test_radas_sign_handler.py diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 4ca1be0d..9d895365 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -423,7 +423,7 @@ def handle_maven_uploading( ) if not _generated_signs: logger.error( - "No sign result files were downloaded, " + "No sign result files were generated, " "please make sure the sign process is already done and without timeout") return (tmp_root, False) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 46794f49..5d25212d 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -103,6 +103,10 @@ def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str] """ Generate .asc files based on RADAS sign result json file """ + if not os.path.isdir(sign_result_loc): + logger.error("Sign result loc dir does not exist: %s", sign_result_loc) + return [], [] + files = [ os.path.join(sign_result_loc, f) for f in os.listdir(sign_result_loc) diff --git a/tests/test_radas_sign_handler.py b/tests/test_radas_sign_handler.py new file mode 100644 index 00000000..f0b90351 --- /dev/null +++ b/tests/test_radas_sign_handler.py @@ -0,0 +1,162 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +import unittest +import tempfile +import os +import json +import shutil +import builtins +from unittest import mock +from charon.utils.files import overwrite_file +from charon.pkgs.radas_signature_handler import generate_radas_sign + +logger = logging.getLogger(__name__) + + +class RadasSignHandlerTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.__prepare_sign_result_file() + + def tearDown(self) -> None: + super().tearDown() + self.__clear_sign_result_file() + + def test_multi_sign_files_generation(self): + self.__prepare_artifacts() + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(len(generated), 2) + self.assertIn(expected_asc1, generated) + self.assertIn(expected_asc2, generated) + + with open(expected_asc1) as f: + content1 = f.read() + with open(expected_asc2) as f: + content2 = f.read() + self.assertIn("signature1@hash", content1) + self.assertIn("signature2@hash", content2) + + def test_sign_files_generation_with_missing_artifacts(self): + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(generated, []) + self.assertFalse(os.path.exists(expected_asc1)) + self.assertFalse(os.path.exists(expected_asc2)) + + def test_sign_files_generation_with_failure(self): + self.__prepare_artifacts() + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + + # simulate expected_asc1 can not open to write properly + real_open = builtins.open + with mock.patch("builtins.open") as mock_open: + def side_effect(path, *args, **kwargs): + # this is for pylint check + mode = "r" + if len(args) > 0: + mode = args[0] + elif "mode" in kwargs: + mode = kwargs["mode"] + if path == expected_asc1 and "w" in mode: + raise IOError("mock write error") + return real_open(path, *args, **kwargs) + mock_open.side_effect = side_effect + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + + self.assertEqual(len(failed), 1) + self.assertNotIn(expected_asc1, generated) + self.assertIn(expected_asc2, generated) + + def test_sign_files_generation_with_missing_result(self): + self.__prepare_artifacts() + # simulate missing pull result by removing the sign result file loc + shutil.rmtree(self.__sign_result_loc) + + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(generated, []) + self.assertFalse(os.path.exists(expected_asc1)) + self.assertFalse(os.path.exists(expected_asc2)) + + def test_sign_files_generation_with_not_single_results(self): + self.__prepare_artifacts() + another_result_file = os.path.join(self.__sign_result_loc, "result2.json") + overwrite_file(another_result_file, "test_json") + + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(generated, []) + self.assertFalse(os.path.exists(expected_asc1)) + self.assertFalse(os.path.exists(expected_asc2)) + + def __prepare_sign_result_file(self): + self.__sign_result_loc = tempfile.mkdtemp() + self.__sign_result_file = os.path.join(self.__sign_result_loc, "result.json") + self.__repo_dir = os.path.join(tempfile.mkdtemp(), "maven-repository") + data = { + "request-id": "request-id", + "file-reference": "quay.io/org/maven-zip@hash", + "result": [ + { + "file": "maven-repository/foo/bar/1.0/foo-bar-1.0.jar", + "signature": ( + "-----BEGIN PGP SIGNATURE-----" + "signature1@hash" + "-----END PGP SIGNATURE-----" + ), + "checksum": "sha256:sha256-content", + }, + { + "file": "maven-repository/foo/bar/2.0/foo-bar-2.0.jar", + "signature": ( + "-----BEGIN PGP SIGNATURE-----" + "signature2@hash" + "-----END PGP SIGNATURE-----" + ), + "checksum": "sha256:sha256-content", + }, + ], + } + json_str = json.dumps(data, indent=2) + overwrite_file(self.__sign_result_file, json_str) + + def __prepare_artifacts(self): + os.makedirs(os.path.join(self.__repo_dir, "foo/bar/1.0"), exist_ok=True) + os.makedirs(os.path.join(self.__repo_dir, "foo/bar/2.0"), exist_ok=True) + artifact1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar") + artifact2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar") + with open(artifact1, "w") as f: + f.write("dummy1") + with open(artifact2, "w") as f: + f.write("dummy2") + + def __clear_sign_result_file(self): + if os.path.exists(self.__sign_result_loc): + shutil.rmtree(self.__sign_result_loc) + if os.path.exists(self.__repo_dir): + shutil.rmtree(self.__repo_dir) From d4b329c154a297b6bf878133644754c18a96075b Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 27 May 2025 11:08:33 +0800 Subject: [PATCH 17/37] Change 'result' to 'results' ref signing/radas-nonprod test samples --- charon/pkgs/radas_signature_handler.py | 2 +- tests/test_radas_sign_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 5d25212d..e670619c 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -160,7 +160,7 @@ async def generate_single_sign_file( failed_paths.append(signature_path) logger.error("Failed to write .asc file for %s: %s", artifact_path, e) - result = data.get("result", []) + result = data.get("results", []) return __do_path_cut_and(path_handler=generate_single_sign_file, data=result) diff --git a/tests/test_radas_sign_handler.py b/tests/test_radas_sign_handler.py index f0b90351..b667eaea 100644 --- a/tests/test_radas_sign_handler.py +++ b/tests/test_radas_sign_handler.py @@ -121,7 +121,7 @@ def __prepare_sign_result_file(self): data = { "request-id": "request-id", "file-reference": "quay.io/org/maven-zip@hash", - "result": [ + "results": [ { "file": "maven-repository/foo/bar/1.0/foo-bar-1.0.jar", "signature": ( From 3639cf012e358ca871895a67ae20bc880dec42f0 Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 28 May 2025 09:36:32 +0800 Subject: [PATCH 18/37] Add request_id match logic for radas message receiver --- charon/pkgs/radas_signature_handler.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index e670619c..24543739 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -37,9 +37,10 @@ class RadasReceiver(MessagingHandler): this value transfers from the cmd flag, should register UmbListener when the client starts """ - def __init__(self, sign_result_loc: str) -> None: + def __init__(self, sign_result_loc: str, request_id: str) -> None: super().__init__() self.sign_result_loc = sign_result_loc + self.request_id = request_id def on_start(self, event: Event) -> None: """ @@ -82,8 +83,19 @@ def _process_message(self, msg: Any) -> None: msg: The message body received """ msg_dict = json.loads(msg) - result_reference_url = msg_dict.get("result_reference") + msg_request_id = msg_dict.get("request_id") + if msg_request_id != self.request_id: + logger.info( + "Message request_id %s does not match the request_id %s from sender, ignoring", + msg_request_id, + self.request_id, + ) + return + logger.info( + "Start to process the sign event message, request_id %s is matched", msg_request_id + ) + result_reference_url = msg_dict.get("result_reference") if not result_reference_url: logger.warning("Not found result_reference in message，ignore.") return From a767747095d8952503f7e69458175c68cfe7672a Mon Sep 17 00:00:00 2001 From: yma Date: Thu, 29 May 2025 11:48:45 +0800 Subject: [PATCH 19/37] Add sign response status and errors for receiver then use to control the main process --- charon/pkgs/radas_signature_handler.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 24543739..a21d75ec 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -19,7 +19,7 @@ import os import asyncio import sys -from typing import List, Any, Tuple, Callable, Dict +from typing import List, Any, Tuple, Callable, Dict, Optional from charon.config import get_config, RadasConfig from charon.pkgs.oras_client import OrasClient from proton import Event @@ -33,14 +33,23 @@ class RadasReceiver(MessagingHandler): This receiver will listen to UMB message queue to receive signing message for signing result. Attributes: - sign_result_loc (str): Local save path (e.g. “/tmp/sign”) for oras pull result, - this value transfers from the cmd flag, should register UmbListener when the client starts + sign_result_loc (str): + Local save path (e.g. “/tmp/sign”) for oras pull result, this value transfers + from the cmd flag,should register UmbListener when the client starts + request_id (str): + Identifier of the request for the signing result + sign_result_status (str): + Result of the signing(success/failed) + sign_result_errors (list): + Any errors encountered if signing fails, this will be empty list if successful """ def __init__(self, sign_result_loc: str, request_id: str) -> None: super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id + self.sign_result_status: Optional[str] = None + self.sign_result_errors: List[str] = [] def on_start(self, event: Event) -> None: """ @@ -95,6 +104,8 @@ def _process_message(self, msg: Any) -> None: logger.info( "Start to process the sign event message, request_id %s is matched", msg_request_id ) + self.sign_result_status = msg_dict.get("signing_status") + self.sign_result_errors = msg_dict.get("errors", []) result_reference_url = msg_dict.get("result_reference") if not result_reference_url: logger.warning("Not found result_reference in message，ignore.") From 124e01d8f2d8610f8c9a4bc5dc463db1dd19fd49 Mon Sep 17 00:00:00 2001 From: liyu Date: Thu, 29 May 2025 17:53:06 +0800 Subject: [PATCH 20/37] Feature of send radas sign request and unit test for it --- charon/pkgs/radas_signature_handler.py | 109 ++++++++++++++++++++++++- tests/test_radas_send_handler.py | 98 ++++++++++++++++++++++ 2 files changed, 203 insertions(+), 4 deletions(-) create mode 100644 tests/test_radas_send_handler.py diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index a21d75ec..6f862d07 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -19,11 +19,13 @@ import os import asyncio import sys +import uuid from typing import List, Any, Tuple, Callable, Dict, Optional from charon.config import get_config, RadasConfig from charon.pkgs.oras_client import OrasClient -from proton import Event +from proton import SSLDomain, Message, Event from proton.handlers import MessagingHandler +from proton.reactor import Container logger = logging.getLogger(__name__) @@ -48,6 +50,7 @@ def __init__(self, sign_result_loc: str, request_id: str) -> None: super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id + self.conn = None self.sign_result_status: Optional[str] = None self.sign_result_errors: List[str] = [] @@ -63,8 +66,23 @@ def on_start(self, event: Event) -> None: # explicit check to pass the type checker if rconf is None: sys.exit(1) - conn = event.container.connect(rconf.umb_target()) - event.container.create_receiver(conn, rconf.result_queue()) + + ssl_domain = SSLDomain(SSLDomain.MODE_CLIENT) + ssl_domain.set_credentials( + rconf.client_ca(), + rconf.client_key(), + rconf.client_key_password() + ) + ssl_domain.set_trusted_ca_db(rconf.root_ca()) + ssl_domain.set_peer_authentication(SSLDomain.VERIFY_PEER) + + self.conn = event.container.connect( + url=rconf.umb_target(), + ssl_domain=ssl_domain + ) + event.container.create_receiver( + self.conn, rconf.result_queue(), dynamic=True + ) logger.info("Listening on %s, queue: %s", rconf.umb_target(), rconf.result_queue()) def on_message(self, event: Event) -> None: @@ -122,6 +140,60 @@ def _process_message(self, msg: Any) -> None: logger.info("Number of files pulled: %d, path: %s", len(files), files[0]) +class RadasSender(MessagingHandler): + """ + This simple sender will send given string massage to UMB message queue to request signing. + Attributes: + payload (str): payload json string for radas to read, + this value construct from the cmd flag + """ + def __init__(self, payload: str): + super().__init__() + self.payload = payload + self.container = None + self.conn = None + self.sender = None + + def on_start(self, event): + """ + On start callback + """ + conf = get_config() + if not (conf and conf.is_radas_enabled()): + sys.exit(1) + + rconf = conf.get_radas_config() + if rconf is None: + sys.exit(1) + + ssl_domain = SSLDomain(SSLDomain.MODE_CLIENT) + ssl_domain.set_credentials( + rconf.client_ca(), + rconf.client_key(), + rconf.client_key_password() + ) + ssl_domain.set_trusted_ca_db(rconf.root_ca()) + ssl_domain.set_peer_authentication(SSLDomain.VERIFY_PEER) + + self.container = event.container + self.conn = event.container.connect( + url=rconf.umb_target(), + ssl_domain=ssl_domain + ) + self.sender = event.container.create_sender(self.conn, rconf.request_queue()) + + def on_sendable(self): + """ + On message able to send callback + """ + request = self.payload + msg = Message(body=request) + if self.sender: + self.sender.send(msg) + if self.container: + self.container.stop() + + def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file @@ -215,4 +287,33 @@ def sign_in_radas(repo_url: str, result_path: str, ignore_patterns: List[str], radas_config: RadasConfig): - logger.info("Start signing for %s", repo_url) + """ + This function will be responsible to do the overall controlling of the whole process, + like trigger the send and register the receiver, and control the wait and timeout there. + """ + logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s," + "radas_config: %s", repo_url, requester, sign_key, result_path, radas_config) + request_id = str(uuid.uuid4()) + exclude = ignore_patterns if ignore_patterns else [] + + payload = { + "request_id": request_id, + "requested_by": requester, + "type": "mrrc", + "file_reference": repo_url, + "sig_keyname": sign_key, + "exclude": exclude + } + + listener = RadasReceiver(result_path, request_id) + sender = RadasSender(json.dumps(payload)) + + try: + Container(sender).run() + logger.info("Successfully sent signing request ID: %s", request_id) + Container(listener).run() + finally: + if listener.conn is not None: + listener.conn.close() + if sender.conn is not None: + sender.conn.close() diff --git a/tests/test_radas_send_handler.py b/tests/test_radas_send_handler.py new file mode 100644 index 00000000..b37fa05a --- /dev/null +++ b/tests/test_radas_send_handler.py @@ -0,0 +1,98 @@ +import tempfile +import os +from unittest import mock +import unittest +from charon.pkgs.radas_signature_handler import sign_in_radas + + +class RadasSignHandlerTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def test_sign_in_radas_normal_flow(self): + with tempfile.TemporaryDirectory() as tmpdir: + # Mock configuration + mock_config = mock.MagicMock() + mock_config.is_radas_enabled.return_value = True + mock_radas_config = mock.MagicMock() + mock_config.get_radas_config.return_value = mock_radas_config + + # Mock Container run to avoid real AMQP connection + with mock.patch( + "charon.pkgs.radas_signature_handler.Container") as mock_container, \ + mock.patch( + "charon.pkgs.radas_signature_handler.get_config", return_value=mock_config), \ + mock.patch( + "charon.pkgs.radas_signature_handler.uuid.uuid4", return_value="mocked-uuid"): + + test_result_path = os.path.join(tmpdir, "results") + os.makedirs(test_result_path) + + sign_in_radas( + repo_url="quay.io/test/repo", + requester="test-user", + sign_key="test-key", + result_path=test_result_path, + ignore_patterns=[], + radas_config=mock_radas_config + ) + + # Verify Container.run() was called twice (sender and receiver) + self.assertEqual(mock_container.call_count, 2) + + # Verify request ID propagation + receiver_call = mock_container.call_args_list[1] + self.assertEqual(receiver_call.args[0].request_id, "mocked-uuid") + + def test_sign_in_radas_with_disabled_config(self): + mock_config = mock.MagicMock() + mock_config.is_radas_enabled.return_value = False + + with mock.patch( + "charon.pkgs.radas_signature_handler.get_config", return_value=mock_config), \ + self.assertRaises(SystemExit): + + sign_in_radas( + repo_url="quay.io/test/repo", + requester="test-user", + sign_key="test-key", + result_path="/tmp/results", + ignore_patterns=[], + radas_config=mock.MagicMock() + ) + + def test_sign_in_radas_connection_cleanup(self): + mock_config = mock.MagicMock() + mock_config.is_radas_enabled.return_value = True + mock_radas_config = mock.MagicMock() + + with mock.patch("charon.pkgs.radas_signature_handler.Container") as mock_container, \ + mock.patch("charon.pkgs.radas_signature_handler.get_config", return_value=mock_config): + + mock_sender_conn = mock.MagicMock() + mock_listener_conn = mock.MagicMock() + + def container_side_effect(*args, **kwargs): + if args[0].__class__.__name__ == "RadasReceiver": + args[0].conn = mock_listener_conn + elif args[0].__class__.__name__ == "RadasSender": + args[0].conn = mock_sender_conn + return mock.MagicMock() + + mock_container.side_effect = container_side_effect + + sign_in_radas( + repo_url="quay.io/test/repo", + requester="test-user", + sign_key="test-key", + result_path="/tmp/results", + ignore_patterns=[], + radas_config=mock_radas_config + ) + + # Verify connections are closed + mock_sender_conn.close.assert_called_once() + mock_listener_conn.close.assert_called_once() From 979ba605492dbd40fcec39ab1bee09157f655f93 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 30 May 2025 15:38:30 +0800 Subject: [PATCH 21/37] Refactor: refactor the RadasSender * rename radas_signature_handler.py to radas_sign.py * change RadasSender logic: * add retry logic * add error handling * avoid duplicate message sending * use a global RadasConfig instead created one in each Radas class --- charon/cmd/cmd_sign.py | 32 +-- charon/config.py | 18 +- charon/pkgs/maven.py | 2 +- ...das_signature_handler.py => radas_sign.py} | 209 ++++++++++-------- pyproject.toml | 3 +- requirements-dev.txt | 1 + setup.py | 3 +- tests/test_radas_send_handler.py | 98 -------- ...ndler.py => test_radas_sign_generation.py} | 2 +- tests/test_radas_sign_sender.py | 86 +++++++ 10 files changed, 236 insertions(+), 218 deletions(-) rename charon/pkgs/{radas_signature_handler.py => radas_sign.py} (61%) delete mode 100644 tests/test_radas_send_handler.py rename tests/{test_radas_sign_handler.py => test_radas_sign_generation.py} (99%) create mode 100644 tests/test_radas_sign_sender.py diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index 629281cb..a83a3e3f 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -16,10 +16,9 @@ from typing import List from charon.config import get_config -from charon.pkgs.radas_signature_handler import sign_in_radas -from charon.cmd.internal import ( - _decide_mode, _safe_delete -) +from charon.pkgs.radas_sign import sign_in_radas +from charon.cmd.internal import _decide_mode + from click import command, option, argument import traceback @@ -39,14 +38,16 @@ "-r", help=""" The requester who sends the signing request. - """ + """, + required=True ) @option( "--result_path", "-p", help=""" The path which will save the sign result file. - """ + """, + required=True ) @option( "--ignore_patterns", @@ -57,14 +58,6 @@ not be allowed to upload to S3. Can accept more than one pattern. """ ) -@option( - "--work_dir", - "-w", - help=""" - The temporary working directory into which archives should - be extracted, when needed. - """ -) @option( "--config", "-c", @@ -79,7 +72,8 @@ help=""" rpm-sign key to be used, will replace {{ key }} in default configuration for signature. Does noting if detach_signature_command does not contain {{ key }} field. - """ + """, + required=True ) @option( "--debug", @@ -100,10 +94,9 @@ def sign( repo_url: str, requester: str, result_path: str, + sign_key: str, ignore_patterns: List[str] = None, - work_dir: str = None, config: str = None, - sign_key: str = "redhatdevel", debug=False, quiet=False, dryrun=False @@ -112,7 +105,6 @@ def sign( radas service. The repo_url points to the maven zip repository in quay.io, which will be sent as the source of the signing. """ - tmp_dir = work_dir logger.debug("%s", ignore_patterns) try: current = datetime.datetime.now().strftime("%Y%m%d%I%M") @@ -139,10 +131,8 @@ def sign( "ignore_patterns": ig_patterns, "radas_config": radas_conf } + logger.debug("params: %s", args) sign_in_radas(**args) # type: ignore except Exception: print(traceback.format_exc()) sys.exit(2) - finally: - if not debug and tmp_dir: - _safe_delete(tmp_dir) diff --git a/charon/config.py b/charon/config.py index 5e7734d0..396e4be3 100644 --- a/charon/config.py +++ b/charon/config.py @@ -75,34 +75,36 @@ def validate(self) -> bool: return True def umb_target(self) -> str: - return f"amqps://{self.__umb_host}:{self.__umb_host_port}" + return f"amqps://{self.__umb_host.strip()}:{self.__umb_host_port}" def result_queue(self) -> str: - return self.__result_queue + return self.__result_queue.strip() def request_queue(self) -> str: - return self.__request_queue + return self.__request_queue.strip() def client_ca(self) -> str: - return self.__client_ca + return self.__client_ca.strip() def client_key(self) -> str: - return self.__client_key + return self.__client_key.strip() def client_key_password(self) -> str: pass_file = self.__client_key_pass_file if os.access(pass_file, os.R_OK): with open(pass_file, "r") as f: - return f.read() + return f.read().strip() elif pass_file: logger.warning("The key password file is not accessible. Will ignore the password.") return "" def root_ca(self) -> str: - return self.__root_ca + return self.__root_ca.strip() def quay_radas_registry_config(self) -> Optional[str]: - return self.__quay_radas_registry_config + if self.__quay_radas_registry_config: + return self.__quay_radas_registry_config.strip() + return None def radas_sign_timeout_retry_count(self) -> int: return self.__radas_sign_timeout_retry_count diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 9d895365..2f525ce3 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -16,7 +16,7 @@ from charon.utils.files import HashType import charon.pkgs.indexing as indexing import charon.pkgs.signature as signature -import charon.pkgs.radas_signature_handler as radas_signature +import charon.pkgs.radas_sign as radas_signature from charon.utils.files import overwrite_file, digest, write_manifest from charon.utils.archive import extract_zip_all from charon.utils.strings import remove_prefix diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_sign.py similarity index 61% rename from charon/pkgs/radas_signature_handler.py rename to charon/pkgs/radas_sign.py index 6f862d07..355daa77 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_sign.py @@ -21,9 +21,9 @@ import sys import uuid from typing import List, Any, Tuple, Callable, Dict, Optional -from charon.config import get_config, RadasConfig +from charon.config import RadasConfig from charon.pkgs.oras_client import OrasClient -from proton import SSLDomain, Message, Event +from proton import SSLDomain, Message, Event, Sender from proton.handlers import MessagingHandler from proton.reactor import Container @@ -46,61 +46,42 @@ class RadasReceiver(MessagingHandler): Any errors encountered if signing fails, this will be empty list if successful """ - def __init__(self, sign_result_loc: str, request_id: str) -> None: + def __init__(self, sign_result_loc: str, request_id: str, rconf: RadasConfig) -> None: super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id self.conn = None self.sign_result_status: Optional[str] = None self.sign_result_errors: List[str] = [] - - def on_start(self, event: Event) -> None: - """ - On start callback - """ - conf = get_config() - if not (conf and conf.is_radas_enabled()): - sys.exit(1) - - rconf = conf.get_radas_config() - # explicit check to pass the type checker - if rconf is None: - sys.exit(1) - - ssl_domain = SSLDomain(SSLDomain.MODE_CLIENT) - ssl_domain.set_credentials( - rconf.client_ca(), - rconf.client_key(), - rconf.client_key_password() + self.rconf = rconf + self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self.ssl.set_trusted_ca_db(self.rconf.root_ca()) + self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self.ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() ) - ssl_domain.set_trusted_ca_db(rconf.root_ca()) - ssl_domain.set_peer_authentication(SSLDomain.VERIFY_PEER) + def on_start(self, event: Event) -> None: self.conn = event.container.connect( - url=rconf.umb_target(), - ssl_domain=ssl_domain + url=self.rconf.umb_target(), + ssl_domain=self.ssl ) event.container.create_receiver( - self.conn, rconf.result_queue(), dynamic=True + self.conn, self.rconf.result_queue(), dynamic=True ) - logger.info("Listening on %s, queue: %s", rconf.umb_target(), rconf.result_queue()) + logger.info("Listening on %s, queue: %s", + self.rconf.umb_target(), + self.rconf.result_queue()) def on_message(self, event: Event) -> None: - """ - On message callback - """ self._process_message(event.message.body) def on_connection_error(self, event: Event) -> None: - """ - On connection error callback - """ logger.error("Received an error event:\n%s", event) def on_disconnected(self, event: Event) -> None: - """ - On disconnected callback - """ logger.error("Disconnected from AMQP broker.") def _process_message(self, msg: Any) -> None: @@ -146,53 +127,106 @@ class RadasSender(MessagingHandler): Attributes: payload (str): payload json string for radas to read, this value construct from the cmd flag + rconf (RadasConfig): the configurations for the radas messaging + system. """ - def __init__(self, payload: str): - super().__init__() + def __init__(self, payload: Any, rconf: RadasConfig): + super(RadasSender, self).__init__() self.payload = payload - self.container = None - self.conn = None - self.sender = None - - def on_start(self, event): - """ - On start callback - """ - conf = get_config() - if not (conf and conf.is_radas_enabled()): - sys.exit(1) - - rconf = conf.get_radas_config() - if rconf is None: - sys.exit(1) - - ssl_domain = SSLDomain(SSLDomain.MODE_CLIENT) - ssl_domain.set_credentials( - rconf.client_ca(), - rconf.client_key(), - rconf.client_key_password() + self.rconf = rconf + self.message_sent = False # Flag to track if message was sent + self.status: Optional[str] = None + self.retried = 0 + self.pending: Optional[Message] = None + self.message: Optional[Message] = None + self.container: Optional[Container] = None + self.sender: Optional[Sender] = None + self.log = logging.getLogger("charon.pkgs.radas_sign.RadasSender") + self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self.ssl.set_trusted_ca_db(self.rconf.root_ca()) + self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self.ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() ) - ssl_domain.set_trusted_ca_db(rconf.root_ca()) - ssl_domain.set_peer_authentication(SSLDomain.VERIFY_PEER) + def on_start(self, event): self.container = event.container - self.conn = event.container.connect( - url=rconf.umb_target(), - ssl_domain=ssl_domain + conn = self.container.connect( + url=self.rconf.umb_target(), + ssl_domain=self.ssl ) - self.sender = event.container.create_sender(self.conn, rconf.request_queue()) - - def on_sendable(self): - """ - On message able to send callback - """ - request = self.payload - msg = Message(body=request) + if conn: + self.sender = self.container.create_sender(conn, self.rconf.request_queue()) + + def on_sendable(self, event): + if not self.message_sent: + msg = Message(body=self.payload, durable=True) + self.log.debug("Sending message: %s to %s", msg.id, event.sender.target.address) + self._send_msg(msg) + self.message = msg + self.message_sent = True + + def on_error(self, event): + self.log.error("Error happened during message sending, reason %s", + event.description) + self.status = "failed" + + def on_rejected(self, event): + self.pending = self.message + self._handle_failed_delivery("Rejected") + + def on_released(self, event): + self.pending = self.message + self._handle_failed_delivery("Released") + + def on_accepted(self, event): + self.log.info("Message accepted by receiver: %s", event.delivery) + self.status = "success" + self.close() # Close connection after confirmation + + def on_timer_task(self, event): + message_to_retry = self.message + self._send_msg(message_to_retry) + self.pending = None + + def close(self): + self.log.info("Message has been sent successfully, close connection") if self.sender: - self.sender.send(msg) + self.sender.close() if self.container: self.container.stop() + def _send_msg(self, msg: Message): + if self.sender and self.sender.credit > 0: + self.sender.send(msg) + self.log.debug("Message %s sent", msg.id) + else: + self.log.warning("Sender not ready or no credit available") + + def _handle_failed_delivery(self, reason: str): + if self.pending: + msg = self.pending + self.log.warning("Message %s failed for reason: %s", msg.id, reason) + max_retries = self.rconf.radas_sign_timeout_retry_count() + if self.retried < max_retries: + # Schedule retry + self.retried = self.retried + 1 + self.log.info("Scheduling retry %s/%s for message %s", + self.retried, max_retries, msg.id) + # Schedule retry after delay + if self.container: + self.container.schedule(self.rconf.radas_sign_timeout_retry_interval(), self) + else: + # Max retries exceeded + self.log.error("Message %s failed after %s retries", msg.id, max_retries) + self.status = "failed" + self.pending = None + else: + self.log.info("Message has been sent successfully, close connection") + self.close() + def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: """ @@ -291,8 +325,8 @@ def sign_in_radas(repo_url: str, This function will be responsible to do the overall controlling of the whole process, like trigger the send and register the receiver, and control the wait and timeout there. """ - logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s," - "radas_config: %s", repo_url, requester, sign_key, result_path, radas_config) + logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s", + repo_url, requester, sign_key, result_path) request_id = str(uuid.uuid4()) exclude = ignore_patterns if ignore_patterns else [] @@ -305,15 +339,16 @@ def sign_in_radas(repo_url: str, "exclude": exclude } - listener = RadasReceiver(result_path, request_id) - sender = RadasSender(json.dumps(payload)) + sender = RadasSender(json.dumps(payload), radas_config) + container = Container(sender) + container.run() - try: - Container(sender).run() - logger.info("Successfully sent signing request ID: %s", request_id) - Container(listener).run() - finally: - if listener.conn is not None: - listener.conn.close() - if sender.conn is not None: - sender.conn.close() + if not sender.status == "success": + logger.error("Something wrong happened in message sending, see logs") + sys.exit(1) + + listener = RadasReceiver(result_path, request_id, radas_config) + Container(listener).run() + + if listener.conn: + listener.conn.close() diff --git a/pyproject.toml b/pyproject.toml index 8e211380..52ddb623 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,8 @@ dependencies = [ "subresource-integrity>=0.2", "jsonschema>=4.9.1", "urllib3>=1.25.10", - "semantic-version>=2.10.0" + "semantic-version>=2.10.0", + "python-qpid-proton>=0.39.0" ] [project.optional-dependencies] diff --git a/requirements-dev.txt b/requirements-dev.txt index f0ed1644..bc38b20f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ -r tests/requirements.txt pyflakes pep8 +tox diff --git a/setup.py b/setup.py index 934ae861..c25ee278 100755 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ "subresource-integrity>=0.2", "jsonschema>=4.9.1", "urllib3>=1.25.10", - "semantic-version>=2.10.0" + "semantic-version>=2.10.0", + "python-qpid-proton>=0.39.0" ], ) diff --git a/tests/test_radas_send_handler.py b/tests/test_radas_send_handler.py deleted file mode 100644 index b37fa05a..00000000 --- a/tests/test_radas_send_handler.py +++ /dev/null @@ -1,98 +0,0 @@ -import tempfile -import os -from unittest import mock -import unittest -from charon.pkgs.radas_signature_handler import sign_in_radas - - -class RadasSignHandlerTest(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - - def tearDown(self) -> None: - super().tearDown() - - def test_sign_in_radas_normal_flow(self): - with tempfile.TemporaryDirectory() as tmpdir: - # Mock configuration - mock_config = mock.MagicMock() - mock_config.is_radas_enabled.return_value = True - mock_radas_config = mock.MagicMock() - mock_config.get_radas_config.return_value = mock_radas_config - - # Mock Container run to avoid real AMQP connection - with mock.patch( - "charon.pkgs.radas_signature_handler.Container") as mock_container, \ - mock.patch( - "charon.pkgs.radas_signature_handler.get_config", return_value=mock_config), \ - mock.patch( - "charon.pkgs.radas_signature_handler.uuid.uuid4", return_value="mocked-uuid"): - - test_result_path = os.path.join(tmpdir, "results") - os.makedirs(test_result_path) - - sign_in_radas( - repo_url="quay.io/test/repo", - requester="test-user", - sign_key="test-key", - result_path=test_result_path, - ignore_patterns=[], - radas_config=mock_radas_config - ) - - # Verify Container.run() was called twice (sender and receiver) - self.assertEqual(mock_container.call_count, 2) - - # Verify request ID propagation - receiver_call = mock_container.call_args_list[1] - self.assertEqual(receiver_call.args[0].request_id, "mocked-uuid") - - def test_sign_in_radas_with_disabled_config(self): - mock_config = mock.MagicMock() - mock_config.is_radas_enabled.return_value = False - - with mock.patch( - "charon.pkgs.radas_signature_handler.get_config", return_value=mock_config), \ - self.assertRaises(SystemExit): - - sign_in_radas( - repo_url="quay.io/test/repo", - requester="test-user", - sign_key="test-key", - result_path="/tmp/results", - ignore_patterns=[], - radas_config=mock.MagicMock() - ) - - def test_sign_in_radas_connection_cleanup(self): - mock_config = mock.MagicMock() - mock_config.is_radas_enabled.return_value = True - mock_radas_config = mock.MagicMock() - - with mock.patch("charon.pkgs.radas_signature_handler.Container") as mock_container, \ - mock.patch("charon.pkgs.radas_signature_handler.get_config", return_value=mock_config): - - mock_sender_conn = mock.MagicMock() - mock_listener_conn = mock.MagicMock() - - def container_side_effect(*args, **kwargs): - if args[0].__class__.__name__ == "RadasReceiver": - args[0].conn = mock_listener_conn - elif args[0].__class__.__name__ == "RadasSender": - args[0].conn = mock_sender_conn - return mock.MagicMock() - - mock_container.side_effect = container_side_effect - - sign_in_radas( - repo_url="quay.io/test/repo", - requester="test-user", - sign_key="test-key", - result_path="/tmp/results", - ignore_patterns=[], - radas_config=mock_radas_config - ) - - # Verify connections are closed - mock_sender_conn.close.assert_called_once() - mock_listener_conn.close.assert_called_once() diff --git a/tests/test_radas_sign_handler.py b/tests/test_radas_sign_generation.py similarity index 99% rename from tests/test_radas_sign_handler.py rename to tests/test_radas_sign_generation.py index b667eaea..33c3d695 100644 --- a/tests/test_radas_sign_handler.py +++ b/tests/test_radas_sign_generation.py @@ -23,7 +23,7 @@ import builtins from unittest import mock from charon.utils.files import overwrite_file -from charon.pkgs.radas_signature_handler import generate_radas_sign +from charon.pkgs.radas_sign import generate_radas_sign logger = logging.getLogger(__name__) diff --git a/tests/test_radas_sign_sender.py b/tests/test_radas_sign_sender.py new file mode 100644 index 00000000..1e75b8fe --- /dev/null +++ b/tests/test_radas_sign_sender.py @@ -0,0 +1,86 @@ +import json +from unittest import mock +import unittest +from charon.pkgs.radas_sign import RadasSender + + +class RadasSignHandlerTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def test_radas_sender(self): + # Mock configuration + mock_radas_config = mock.MagicMock() + mock_radas_config.validate.return_value = True + mock_radas_config.client_ca.return_value = "test-client-ca" + mock_radas_config.client_key.return_value = "test-client-key" + mock_radas_config.client_key_password.return_value = "test-client-key-pass" + mock_radas_config.root_ca.return_value = "test-root-ca" + mock_radas_config.radas_sign_timeout_retry_count.return_value = 5 + + test_payload = { + "request_id": "mock-id", + "requested_by": "test-user", + "type": "mrrc", + "file_reference": "quay.io/test/repo", + "sig_keyname": "test-key", + "exclude": [] + } + + # Mock Container run to avoid real AMQP connection + with mock.patch( + "charon.pkgs.radas_sign.Container") as mock_container, \ + mock.patch("charon.pkgs.radas_sign.SSLDomain") as ssl_domain, \ + mock.patch("charon.pkgs.radas_sign.Event") as event: + + json_payload = json.dumps(test_payload) + r_sender = RadasSender(json_payload, mock_radas_config) + self.assertEqual(ssl_domain.call_count, 1) + self.assertEqual(r_sender.payload, json_payload) + self.assertIs(r_sender.rconf, mock_radas_config) + self.assertIsNone(r_sender.message) + self.assertIsNone(r_sender.pending) + + # test on_start + mock_sender = mock.MagicMock() + mock_conn = mock.MagicMock() + mock_container.connect.return_value = mock_conn + mock_container.create_sender.return_value = mock_sender + event.container = mock_container + r_sender.on_start(event) + self.assertEqual(mock_container.connect.call_count, 1) + self.assertEqual(mock_container.create_sender.call_count, 1) + + # test on_sendable + mock_sender.credit = 1 + r_sender.on_sendable(event) + self.assertIsNotNone(r_sender.message) + self.assertEqual(mock_sender.send.call_count, 1) + + # test on_accepted + r_sender.on_accepted(event) + self.assertEqual(r_sender.status, "success") + self.assertEqual(r_sender.retried, 0) + self.assertEqual(r_sender.sender.close.call_count, 1) + self.assertEqual(r_sender.container.stop.call_count, 1) + + # test on_rejected + r_sender.on_rejected(event) + self.assertIsNone(r_sender.pending) + self.assertEqual(r_sender.retried, 1) + self.assertEqual(r_sender.container.schedule.call_count, 1) + + # test on_released + r_sender.on_released(event) + self.assertIsNone(r_sender.pending) + self.assertEqual(r_sender.retried, 2) + self.assertEqual(r_sender.container.schedule.call_count, 2) + + # test on_released + r_sender.on_timer_task(event) + self.assertIsNone(r_sender.pending) + self.assertEqual(r_sender.retried, 2) + self.assertEqual(mock_sender.send.call_count, 2) From 087995686dab0f12e8e1ac30763346a5c2f5d2b3 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 5 Jun 2025 10:52:15 +0800 Subject: [PATCH 22/37] Refactor: Refactor the RadasReceiver * Support timeout mechanism for Receiver exit --- charon/config.py | 4 + charon/pkgs/radas_sign.py | 113 ++++++++++++++++++-------- tests/test_radas_sign_receiver.py | 130 ++++++++++++++++++++++++++++++ tests/test_radas_sign_sender.py | 2 +- 4 files changed, 214 insertions(+), 35 deletions(-) create mode 100644 tests/test_radas_sign_receiver.py diff --git a/charon/config.py b/charon/config.py index 396e4be3..39e9dc80 100644 --- a/charon/config.py +++ b/charon/config.py @@ -42,6 +42,7 @@ def __init__(self, data: Dict): self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 ) + self.__radas_receiver_timeout: int = int(data.get("radas_receiver_timeout", 1800)) def validate(self) -> bool: if not self.__umb_host: @@ -112,6 +113,9 @@ def radas_sign_timeout_retry_count(self) -> int: def radas_sign_timeout_retry_interval(self) -> int: return self.__radas_sign_timeout_retry_interval + def receiver_timeout(self) -> int: + return self.__radas_receiver_timeout + class CharonConfig(object): """CharonConfig is used to store all configurations for charon diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 355daa77..0ac558b8 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -17,13 +17,14 @@ import logging import json import os -import asyncio import sys +import asyncio import uuid +import time from typing import List, Any, Tuple, Callable, Dict, Optional from charon.config import RadasConfig from charon.pkgs.oras_client import OrasClient -from proton import SSLDomain, Message, Event, Sender +from proton import SSLDomain, Message, Event, Sender, Connection from proton.handlers import MessagingHandler from proton.reactor import Container @@ -40,6 +41,8 @@ class RadasReceiver(MessagingHandler): from the cmd flag,should register UmbListener when the client starts request_id (str): Identifier of the request for the signing result + rconf (RadasConfig): + the configurations for the radas messaging system. sign_result_status (str): Result of the signing(success/failed) sign_result_errors (list): @@ -50,10 +53,13 @@ def __init__(self, sign_result_loc: str, request_id: str, rconf: RadasConfig) -> super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id - self.conn = None + self.conn: Optional[Connection] = None + self.message_handled = False self.sign_result_status: Optional[str] = None self.sign_result_errors: List[str] = [] self.rconf = rconf + self.start_time = 0.0 + self.timeout_check_delay = 30.0 self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) self.ssl.set_trusted_ca_db(self.rconf.root_ca()) self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) @@ -62,27 +68,58 @@ def __init__(self, sign_result_loc: str, request_id: str, rconf: RadasConfig) -> self.rconf.client_key(), self.rconf.client_key_password() ) + self.log = logging.getLogger("charon.pkgs.radas_sign.RadasReceiver") def on_start(self, event: Event) -> None: - self.conn = event.container.connect( - url=self.rconf.umb_target(), - ssl_domain=self.ssl + umb_target = self.rconf.umb_target() + container = event.container + self.conn = container.connect( + url=umb_target, + ssl_domain=self.ssl, + heartbeat=500 ) - event.container.create_receiver( - self.conn, self.rconf.result_queue(), dynamic=True + receiver = container.create_receiver( + context=self.conn, source=self.rconf.result_queue(), ) - logger.info("Listening on %s, queue: %s", - self.rconf.umb_target(), - self.rconf.result_queue()) + self.log.info("Listening on %s, queue: %s", + umb_target, + receiver.source.address) + self.start_time = time.time() + container.schedule(self.timeout_check_delay, self) + + def on_timer_task(self, event: Event) -> None: + current = time.time() + timeout = self.rconf.receiver_timeout() + idle_time = current - self.start_time + self.log.debug("Checking timeout: passed %s seconds, timeout time %s seconds", + idle_time, timeout) + if idle_time > self.rconf.receiver_timeout(): + self.log.error("The receiver did not receive messages for more than %s seconds," + " and needs to stop receiving and quit.", timeout) + self._close(event) + else: + event.container.schedule(self.timeout_check_delay, self) def on_message(self, event: Event) -> None: + self.log.debug("Got message: %s", event.message.body) self._process_message(event.message.body) + if self.message_handled: + self.log.debug("The signing result is handled.") + self._close(event) - def on_connection_error(self, event: Event) -> None: - logger.error("Received an error event:\n%s", event) + def on_error(self, event: Event) -> None: + self.log.error("Received an error event:\n%s", event.message.body) def on_disconnected(self, event: Event) -> None: - logger.error("Disconnected from AMQP broker.") + self.log.info("Disconnected from AMQP broker: %s", + event.connection.connected_address) + + def _close(self, event: Event) -> None: + if event: + if event.connection: + event.connection.close() + if event.container: + event.container.stop() def _process_message(self, msg: Any) -> None: """ @@ -93,32 +130,37 @@ def _process_message(self, msg: Any) -> None: msg_dict = json.loads(msg) msg_request_id = msg_dict.get("request_id") if msg_request_id != self.request_id: - logger.info( + self.log.info( "Message request_id %s does not match the request_id %s from sender, ignoring", msg_request_id, self.request_id, ) return - logger.info( + self.message_handled = True + self.log.info( "Start to process the sign event message, request_id %s is matched", msg_request_id ) self.sign_result_status = msg_dict.get("signing_status") self.sign_result_errors = msg_dict.get("errors", []) - result_reference_url = msg_dict.get("result_reference") - if not result_reference_url: - logger.warning("Not found result_reference in message，ignore.") - return + if self.sign_result_status == "success": + result_reference_url = msg_dict.get("result_reference") + if not result_reference_url: + self.log.warning("Not found result_reference in message，ignore.") + return - logger.info("Using SIGN RESULT LOC: %s", self.sign_result_loc) - sign_result_parent_dir = os.path.dirname(self.sign_result_loc) - os.makedirs(sign_result_parent_dir, exist_ok=True) + self.log.info("Using SIGN RESULT LOC: %s", self.sign_result_loc) + sign_result_parent_dir = os.path.dirname(self.sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) - oras_client = OrasClient() - files = oras_client.pull( - result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc - ) - logger.info("Number of files pulled: %d, path: %s", len(files), files[0]) + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc + ) + self.log.info("Number of files pulled: %d, path: %s", len(files), files[0]) + else: + self.log.error("The signing result received with failed status. Errors: %s", + self.sign_result_errors) class RadasSender(MessagingHandler): @@ -141,7 +183,6 @@ def __init__(self, payload: Any, rconf: RadasConfig): self.message: Optional[Message] = None self.container: Optional[Container] = None self.sender: Optional[Sender] = None - self.log = logging.getLogger("charon.pkgs.radas_sign.RadasSender") self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) self.ssl.set_trusted_ca_db(self.rconf.root_ca()) self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) @@ -150,6 +191,7 @@ def __init__(self, payload: Any, rconf: RadasConfig): self.rconf.client_key(), self.rconf.client_key_password() ) + self.log = logging.getLogger("charon.pkgs.radas_sign.RadasSender") def on_start(self, event): self.container = event.container @@ -329,7 +371,6 @@ def sign_in_radas(repo_url: str, repo_url, requester, sign_key, result_path) request_id = str(uuid.uuid4()) exclude = ignore_patterns if ignore_patterns else [] - payload = { "request_id": request_id, "requested_by": requester, @@ -347,8 +388,12 @@ def sign_in_radas(repo_url: str, logger.error("Something wrong happened in message sending, see logs") sys.exit(1) - listener = RadasReceiver(result_path, request_id, radas_config) - Container(listener).run() + # request_id = "some-request-id-1" # for test purpose + receiver = RadasReceiver(result_path, request_id, radas_config) + Container(receiver).run() - if listener.conn: - listener.conn.close() + status = receiver.sign_result_status + if status != "success": + logger.error("The signing result is processed with errors: %s", + receiver.sign_result_errors) + sys.exit(1) diff --git a/tests/test_radas_sign_receiver.py b/tests/test_radas_sign_receiver.py new file mode 100644 index 00000000..c75a8e64 --- /dev/null +++ b/tests/test_radas_sign_receiver.py @@ -0,0 +1,130 @@ +from unittest import mock +import unittest +import tempfile +import time +import json +from charon.pkgs.radas_sign import RadasReceiver + + +class RadasSignReceiverTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def reset_receiver(self, r_receiver: RadasReceiver) -> None: + r_receiver.message_handled = False + r_receiver.sign_result_errors = [] + r_receiver.sign_result_status = None + + def test_radas_receiver(self): + # Mock configuration + mock_radas_config = mock.MagicMock() + mock_radas_config.validate.return_value = True + mock_radas_config.client_ca.return_value = "test-client-ca" + mock_radas_config.client_key.return_value = "test-client-key" + mock_radas_config.client_key_password.return_value = "test-client-key-pass" + mock_radas_config.root_ca.return_value = "test-root-ca" + mock_radas_config.receiver_timeout.return_value = 60 + + # Mock Container run to avoid real AMQP connection + with mock.patch( + "charon.pkgs.radas_sign.Container") as mock_container, \ + mock.patch("charon.pkgs.radas_sign.SSLDomain") as ssl_domain, \ + mock.patch("charon.pkgs.radas_sign.OrasClient") as oras_client, \ + mock.patch("charon.pkgs.radas_sign.Event") as event: + test_result_path = tempfile.mkdtemp() + test_request_id = "test-request-id" + r_receiver = RadasReceiver(test_result_path, test_request_id, mock_radas_config) + self.assertEqual(ssl_domain.call_count, 1) + self.assertEqual(r_receiver.sign_result_loc, test_result_path) + self.assertEqual(r_receiver.request_id, test_request_id) + + # prepare mock + mock_receiver = mock.MagicMock() + mock_conn = mock.MagicMock() + mock_container.connect.return_value = mock_conn + mock_container.create_receiver.return_value = mock_receiver + event.container = mock_container + event.message = mock.MagicMock() + event.connection = mock.MagicMock() + + # test on_start + r_receiver.on_start(event) + self.assertEqual(mock_container.connect.call_count, 1) + self.assertEqual(mock_container.create_receiver.call_count, 1) + self.assertTrue(r_receiver.start_time > 0.0) + self.assertTrue(r_receiver.start_time < time.time()) + self.assertEqual(mock_container.schedule.call_count, 1) + + # test on_message: unmatched case + test_ummatch_result = { + "request_id": "test-request-id-no-match", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } + event.message.body = json.dumps(test_ummatch_result) + r_receiver.on_message(event) + self.assertEqual(event.connection.close.call_count, 0) + self.assertEqual(mock_container.stop.call_count, 0) + self.assertFalse(r_receiver.message_handled) + self.assertIsNone(r_receiver.sign_result_status) + self.assertEqual(r_receiver.sign_result_errors, []) + self.assertEqual(oras_client.call_count, 0) + + # test on_message: matched case with failed status + self.reset_receiver(r_receiver) + test_failed_result = { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "failed", + "errors": ["error1", "error2"] + } + event.message.body = json.dumps(test_failed_result) + r_receiver.on_message(event) + self.assertEqual(event.connection.close.call_count, 1) + self.assertEqual(mock_container.stop.call_count, 1) + self.assertTrue(r_receiver.message_handled) + self.assertEqual(r_receiver.sign_result_status, "failed") + self.assertEqual(r_receiver.sign_result_errors, ["error1", "error2"]) + self.assertEqual(oras_client.call_count, 0) + + # test on_message: matched case with success status + self.reset_receiver(r_receiver) + test_success_result = { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } + event.message.body = json.dumps(test_success_result) + r_receiver.on_message(event) + self.assertEqual(event.connection.close.call_count, 2) + self.assertEqual(mock_container.stop.call_count, 2) + self.assertTrue(r_receiver.message_handled) + self.assertEqual(r_receiver.sign_result_status, "success") + self.assertEqual(r_receiver.sign_result_errors, []) + self.assertEqual(oras_client.call_count, 1) + oras_client_call = oras_client.return_value + self.assertEqual(oras_client_call.pull.call_count, 1) + + # test on_timer_task: not timeout + r_receiver.on_timer_task(event) + self.assertEqual(event.connection.close.call_count, 2) + self.assertEqual(mock_container.stop.call_count, 2) + self.assertEqual(mock_container.schedule.call_count, 2) + + # test on_timer_task: timeout + mock_radas_config.receiver_timeout.return_value = 0 + r_receiver.on_timer_task(event) + self.assertEqual(event.connection.close.call_count, 3) + self.assertEqual(mock_container.stop.call_count, 3) + self.assertEqual(mock_container.schedule.call_count, 2) diff --git a/tests/test_radas_sign_sender.py b/tests/test_radas_sign_sender.py index 1e75b8fe..602d7de6 100644 --- a/tests/test_radas_sign_sender.py +++ b/tests/test_radas_sign_sender.py @@ -4,7 +4,7 @@ from charon.pkgs.radas_sign import RadasSender -class RadasSignHandlerTest(unittest.TestCase): +class RadasSignSenderTest(unittest.TestCase): def setUp(self) -> None: super().setUp() From 7ec924abc0b0c7a5a07c38af3e170b59e5de841c Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 5 Jun 2025 16:07:21 +0800 Subject: [PATCH 23/37] Some chore changes: * Add ssl enable condition in RadasConfig * Mark some fields as "private" for RadasSender and RadasReceiver --- charon/cmd/cmd_sign.py | 2 - charon/config.py | 3 + charon/pkgs/radas_sign.py | 126 ++++++++++++++++-------------- tests/test_radas_sign_receiver.py | 12 +-- tests/test_radas_sign_sender.py | 28 +++---- 5 files changed, 89 insertions(+), 82 deletions(-) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index a83a3e3f..f17fe028 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -109,8 +109,6 @@ def sign( try: current = datetime.datetime.now().strftime("%Y%m%d%I%M") _decide_mode("radas_sign", current, is_quiet=quiet, is_debug=debug) - if dryrun: - logger.info("Running in dry-run mode, no files will signed.") conf = get_config(config) if not conf: logger.error("The charon configuration is not valid!") diff --git a/charon/config.py b/charon/config.py index 39e9dc80..a99c1436 100644 --- a/charon/config.py +++ b/charon/config.py @@ -102,6 +102,9 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca.strip() + def ssl_enabled(self) -> bool: + return bool(self.__client_ca and self.__client_key and self.__root_ca) + def quay_radas_registry_config(self) -> Optional[str]: if self.__quay_radas_registry_config: return self.__quay_radas_registry_config.strip() diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 0ac558b8..d5c50cca 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -53,44 +53,46 @@ def __init__(self, sign_result_loc: str, request_id: str, rconf: RadasConfig) -> super().__init__() self.sign_result_loc = sign_result_loc self.request_id = request_id - self.conn: Optional[Connection] = None - self.message_handled = False self.sign_result_status: Optional[str] = None self.sign_result_errors: List[str] = [] self.rconf = rconf - self.start_time = 0.0 - self.timeout_check_delay = 30.0 - self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) - self.ssl.set_trusted_ca_db(self.rconf.root_ca()) - self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) - self.ssl.set_credentials( - self.rconf.client_ca(), - self.rconf.client_key(), - self.rconf.client_key_password() - ) + self._conn: Optional[Connection] = None + self._message_handled = False + self._start_time = 0.0 + self._timeout_check_delay = 30.0 + self._ssl: Optional[SSLDomain] = None + if rconf.ssl_enabled(): + self._ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self._ssl.set_trusted_ca_db(self.rconf.root_ca()) + self._ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self._ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() + ) self.log = logging.getLogger("charon.pkgs.radas_sign.RadasReceiver") def on_start(self, event: Event) -> None: umb_target = self.rconf.umb_target() container = event.container - self.conn = container.connect( + self._conn = container.connect( url=umb_target, - ssl_domain=self.ssl, + ssl_domain=self._ssl, heartbeat=500 ) receiver = container.create_receiver( - context=self.conn, source=self.rconf.result_queue(), + context=self._conn, source=self.rconf.result_queue(), ) self.log.info("Listening on %s, queue: %s", umb_target, receiver.source.address) - self.start_time = time.time() - container.schedule(self.timeout_check_delay, self) + self._start_time = time.time() + container.schedule(self._timeout_check_delay, self) def on_timer_task(self, event: Event) -> None: current = time.time() timeout = self.rconf.receiver_timeout() - idle_time = current - self.start_time + idle_time = current - self._start_time self.log.debug("Checking timeout: passed %s seconds, timeout time %s seconds", idle_time, timeout) if idle_time > self.rconf.receiver_timeout(): @@ -98,12 +100,12 @@ def on_timer_task(self, event: Event) -> None: " and needs to stop receiving and quit.", timeout) self._close(event) else: - event.container.schedule(self.timeout_check_delay, self) + event.container.schedule(self._timeout_check_delay, self) def on_message(self, event: Event) -> None: self.log.debug("Got message: %s", event.message.body) self._process_message(event.message.body) - if self.message_handled: + if self._message_handled: self.log.debug("The signing result is handled.") self._close(event) @@ -137,7 +139,7 @@ def _process_message(self, msg: Any) -> None: ) return - self.message_handled = True + self._message_handled = True self.log.info( "Start to process the sign event message, request_id %s is matched", msg_request_id ) @@ -171,44 +173,48 @@ class RadasSender(MessagingHandler): this value construct from the cmd flag rconf (RadasConfig): the configurations for the radas messaging system. + status (str): tell if status for message sending, only "success" + means the message is sent successfully. """ def __init__(self, payload: Any, rconf: RadasConfig): super(RadasSender, self).__init__() self.payload = payload self.rconf = rconf - self.message_sent = False # Flag to track if message was sent self.status: Optional[str] = None - self.retried = 0 - self.pending: Optional[Message] = None - self.message: Optional[Message] = None - self.container: Optional[Container] = None - self.sender: Optional[Sender] = None - self.ssl = SSLDomain(SSLDomain.MODE_CLIENT) - self.ssl.set_trusted_ca_db(self.rconf.root_ca()) - self.ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) - self.ssl.set_credentials( - self.rconf.client_ca(), - self.rconf.client_key(), - self.rconf.client_key_password() - ) + self._message_sent = False # Flag to track if message was sent + self._retried = 0 + self._pending: Optional[Message] = None + self._message: Optional[Message] = None + self._container: Optional[Container] = None + self._sender: Optional[Sender] = None + self._ssl: Optional[SSLDomain] = None + if self.rconf.ssl_enabled(): + self._ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self._ssl.set_trusted_ca_db(self.rconf.root_ca()) + self._ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self._ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() + ) self.log = logging.getLogger("charon.pkgs.radas_sign.RadasSender") def on_start(self, event): - self.container = event.container - conn = self.container.connect( + self._container = event.container + conn = self._container.connect( url=self.rconf.umb_target(), - ssl_domain=self.ssl + ssl_domain=self._ssl ) if conn: - self.sender = self.container.create_sender(conn, self.rconf.request_queue()) + self._sender = self._container.create_sender(conn, self.rconf.request_queue()) def on_sendable(self, event): - if not self.message_sent: + if not self._message_sent: msg = Message(body=self.payload, durable=True) self.log.debug("Sending message: %s to %s", msg.id, event.sender.target.address) self._send_msg(msg) - self.message = msg - self.message_sent = True + self._message = msg + self._message_sent = True def on_error(self, event): self.log.error("Error happened during message sending, reason %s", @@ -216,11 +222,11 @@ def on_error(self, event): self.status = "failed" def on_rejected(self, event): - self.pending = self.message + self._pending = self._message self._handle_failed_delivery("Rejected") def on_released(self, event): - self.pending = self.message + self._pending = self._message self._handle_failed_delivery("Released") def on_accepted(self, event): @@ -229,42 +235,42 @@ def on_accepted(self, event): self.close() # Close connection after confirmation def on_timer_task(self, event): - message_to_retry = self.message + message_to_retry = self._message self._send_msg(message_to_retry) - self.pending = None + self._pending = None def close(self): self.log.info("Message has been sent successfully, close connection") - if self.sender: - self.sender.close() - if self.container: - self.container.stop() + if self._sender: + self._sender.close() + if self._container: + self._container.stop() def _send_msg(self, msg: Message): - if self.sender and self.sender.credit > 0: - self.sender.send(msg) + if self._sender and self._sender.credit > 0: + self._sender.send(msg) self.log.debug("Message %s sent", msg.id) else: self.log.warning("Sender not ready or no credit available") def _handle_failed_delivery(self, reason: str): - if self.pending: - msg = self.pending + if self._pending: + msg = self._pending self.log.warning("Message %s failed for reason: %s", msg.id, reason) max_retries = self.rconf.radas_sign_timeout_retry_count() - if self.retried < max_retries: + if self._retried < max_retries: # Schedule retry - self.retried = self.retried + 1 + self._retried = self._retried + 1 self.log.info("Scheduling retry %s/%s for message %s", - self.retried, max_retries, msg.id) + self._retried, max_retries, msg.id) # Schedule retry after delay - if self.container: - self.container.schedule(self.rconf.radas_sign_timeout_retry_interval(), self) + if self._container: + self._container.schedule(self.rconf.radas_sign_timeout_retry_interval(), self) else: # Max retries exceeded self.log.error("Message %s failed after %s retries", msg.id, max_retries) self.status = "failed" - self.pending = None + self._pending = None else: self.log.info("Message has been sent successfully, close connection") self.close() diff --git a/tests/test_radas_sign_receiver.py b/tests/test_radas_sign_receiver.py index c75a8e64..e80f0435 100644 --- a/tests/test_radas_sign_receiver.py +++ b/tests/test_radas_sign_receiver.py @@ -14,7 +14,7 @@ def tearDown(self) -> None: super().tearDown() def reset_receiver(self, r_receiver: RadasReceiver) -> None: - r_receiver.message_handled = False + r_receiver._message_handled = False r_receiver.sign_result_errors = [] r_receiver.sign_result_status = None @@ -54,8 +54,8 @@ def test_radas_receiver(self): r_receiver.on_start(event) self.assertEqual(mock_container.connect.call_count, 1) self.assertEqual(mock_container.create_receiver.call_count, 1) - self.assertTrue(r_receiver.start_time > 0.0) - self.assertTrue(r_receiver.start_time < time.time()) + self.assertTrue(r_receiver._start_time > 0.0) + self.assertTrue(r_receiver._start_time < time.time()) self.assertEqual(mock_container.schedule.call_count, 1) # test on_message: unmatched case @@ -71,7 +71,7 @@ def test_radas_receiver(self): r_receiver.on_message(event) self.assertEqual(event.connection.close.call_count, 0) self.assertEqual(mock_container.stop.call_count, 0) - self.assertFalse(r_receiver.message_handled) + self.assertFalse(r_receiver._message_handled) self.assertIsNone(r_receiver.sign_result_status) self.assertEqual(r_receiver.sign_result_errors, []) self.assertEqual(oras_client.call_count, 0) @@ -90,7 +90,7 @@ def test_radas_receiver(self): r_receiver.on_message(event) self.assertEqual(event.connection.close.call_count, 1) self.assertEqual(mock_container.stop.call_count, 1) - self.assertTrue(r_receiver.message_handled) + self.assertTrue(r_receiver._message_handled) self.assertEqual(r_receiver.sign_result_status, "failed") self.assertEqual(r_receiver.sign_result_errors, ["error1", "error2"]) self.assertEqual(oras_client.call_count, 0) @@ -109,7 +109,7 @@ def test_radas_receiver(self): r_receiver.on_message(event) self.assertEqual(event.connection.close.call_count, 2) self.assertEqual(mock_container.stop.call_count, 2) - self.assertTrue(r_receiver.message_handled) + self.assertTrue(r_receiver._message_handled) self.assertEqual(r_receiver.sign_result_status, "success") self.assertEqual(r_receiver.sign_result_errors, []) self.assertEqual(oras_client.call_count, 1) diff --git a/tests/test_radas_sign_sender.py b/tests/test_radas_sign_sender.py index 602d7de6..c1c1fee3 100644 --- a/tests/test_radas_sign_sender.py +++ b/tests/test_radas_sign_sender.py @@ -41,8 +41,8 @@ def test_radas_sender(self): self.assertEqual(ssl_domain.call_count, 1) self.assertEqual(r_sender.payload, json_payload) self.assertIs(r_sender.rconf, mock_radas_config) - self.assertIsNone(r_sender.message) - self.assertIsNone(r_sender.pending) + self.assertIsNone(r_sender._message) + self.assertIsNone(r_sender._pending) # test on_start mock_sender = mock.MagicMock() @@ -57,30 +57,30 @@ def test_radas_sender(self): # test on_sendable mock_sender.credit = 1 r_sender.on_sendable(event) - self.assertIsNotNone(r_sender.message) + self.assertIsNotNone(r_sender._message) self.assertEqual(mock_sender.send.call_count, 1) # test on_accepted r_sender.on_accepted(event) self.assertEqual(r_sender.status, "success") - self.assertEqual(r_sender.retried, 0) - self.assertEqual(r_sender.sender.close.call_count, 1) - self.assertEqual(r_sender.container.stop.call_count, 1) + self.assertEqual(r_sender._retried, 0) + self.assertEqual(r_sender._sender.close.call_count, 1) + self.assertEqual(r_sender._container.stop.call_count, 1) # test on_rejected r_sender.on_rejected(event) - self.assertIsNone(r_sender.pending) - self.assertEqual(r_sender.retried, 1) - self.assertEqual(r_sender.container.schedule.call_count, 1) + self.assertIsNone(r_sender._pending) + self.assertEqual(r_sender._retried, 1) + self.assertEqual(r_sender._container.schedule.call_count, 1) # test on_released r_sender.on_released(event) - self.assertIsNone(r_sender.pending) - self.assertEqual(r_sender.retried, 2) - self.assertEqual(r_sender.container.schedule.call_count, 2) + self.assertIsNone(r_sender._pending) + self.assertEqual(r_sender._retried, 2) + self.assertEqual(r_sender._container.schedule.call_count, 2) # test on_released r_sender.on_timer_task(event) - self.assertIsNone(r_sender.pending) - self.assertEqual(r_sender.retried, 2) + self.assertIsNone(r_sender._pending) + self.assertEqual(r_sender._retried, 2) self.assertEqual(mock_sender.send.call_count, 2) From 21e69669bf4c11a0e47d4389f534b11ebc9374b0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 5 Jun 2025 20:29:27 +0800 Subject: [PATCH 24/37] RADAS: change the request_queue to request_channel in config As topic is also a valid destination to send, use queue is not correct --- charon/config.py | 8 ++++---- charon/pkgs/radas_sign.py | 2 +- tests/test_config_radas.py | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/charon/config.py b/charon/config.py index a99c1436..d98bb995 100644 --- a/charon/config.py +++ b/charon/config.py @@ -30,7 +30,7 @@ def __init__(self, data: Dict): self.__umb_host: str = data.get("umb_host", None) self.__umb_host_port: str = data.get("umb_host_port", "5671") self.__result_queue: str = data.get("result_queue", None) - self.__request_queue: str = data.get("request_queue", None) + self.__request_chan: str = data.get("request_channel", None) self.__client_ca: str = data.get("client_ca", None) self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) @@ -51,7 +51,7 @@ def validate(self) -> bool: if not self.__result_queue: logger.error("Missing the queue setting to receive signing result in UMB!") return False - if not self.__request_queue: + if not self.__request_chan: logger.error("Missing the queue setting to send signing request in UMB!") return False if self.__client_ca and not os.access(self.__client_ca, os.R_OK): @@ -81,8 +81,8 @@ def umb_target(self) -> str: def result_queue(self) -> str: return self.__result_queue.strip() - def request_queue(self) -> str: - return self.__request_queue.strip() + def request_channel(self) -> str: + return self.__request_chan.strip() def client_ca(self) -> str: return self.__client_ca.strip() diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index d5c50cca..7ac4e3dc 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -206,7 +206,7 @@ def on_start(self, event): ssl_domain=self._ssl ) if conn: - self._sender = self._container.create_sender(conn, self.rconf.request_queue()) + self._sender = self._container.create_sender(conn, self.rconf.request_channel()) def on_sendable(self, event): if not self._message_sent: diff --git a/tests/test_config_radas.py b/tests/test_config_radas.py index 152dc1c2..a6c7d5a4 100644 --- a/tests/test_config_radas.py +++ b/tests/test_config_radas.py @@ -36,7 +36,7 @@ def test_full_radas_config(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -59,7 +59,7 @@ def test_missing_umb_host(self): radas_settings = """ radas: result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -79,7 +79,7 @@ def test_missing_result_queue(self): radas_settings = """ radas: umb_host: test.umb.api.com - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -120,7 +120,7 @@ def test_unaccessible_client_ca(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -142,7 +142,7 @@ def test_unaccessible_client_key(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -164,7 +164,7 @@ def test_unaccessible_client_password_file(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} @@ -186,7 +186,7 @@ def test_unaccessible_root_ca(self): radas: umb_host: test.umb.api.com result_queue: queue.result.test - request_queue: queue.request.test + request_channel: topic://topic.request.test client_ca: {} client_key: {} client_key_pass_file: {} From de5be71c6e5d99512c3788a729caa608037e2788 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 6 Jun 2025 18:22:27 +0800 Subject: [PATCH 25/37] Add missing oras deps in project.toml and setup.py --- pyproject.toml | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 52ddb623..c5081bbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "jsonschema>=4.9.1", "urllib3>=1.25.10", "semantic-version>=2.10.0", + "oras>=0.2.31", "python-qpid-proton>=0.39.0" ] diff --git a/setup.py b/setup.py index c25ee278..29b6833e 100755 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ "jsonschema>=4.9.1", "urllib3>=1.25.10", "semantic-version>=2.10.0", + "oras>=0.2.31", "python-qpid-proton>=0.39.0" ], ) From 4b1f5df21883a7ac8322b947155d79ece632c20c Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 9 Jun 2025 15:17:37 +0800 Subject: [PATCH 26/37] RADAS: update Containerfile for radas support * Update base image to new version of ubi8-minimal * Add gcc, openssl-devel as dependencies for build * Use python12-devel instead of python12 because qpid-proton --- image/Containerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/image/Containerfile b/image/Containerfile index 267cb9f0..a84a8da9 100644 --- a/image/Containerfile +++ b/image/Containerfile @@ -19,16 +19,16 @@ # 4. Start using uploader # charon upload/delete from /home/charon/upload/... ### -FROM registry.access.redhat.com/ubi8-minimal:8.10-1052 as builder +FROM registry.access.redhat.com/ubi8-minimal:8.10-1295 as builder ARG GIT_BRANCH=release -RUN microdnf install -y git-core python3.12 python3.12-pip && microdnf clean all +RUN microdnf install -y git-core python3.12-devel python3.12-pip gcc openssl-devel && microdnf clean all RUN git clone -b ${GIT_BRANCH} --depth 1 https://github.com/Commonjava/charon.git RUN pip3 install --no-cache-dir --upgrade pip RUN pip3 wheel ./charon -FROM registry.access.redhat.com/ubi8-minimal:8.10-1052 +FROM registry.access.redhat.com/ubi8-minimal:8.10-1295 ARG USER=charon ARG UID=10000 @@ -38,7 +38,7 @@ WORKDIR ${HOME_DIR} USER root -RUN microdnf install -y python3.12 python3.12-pip shadow-utils && microdnf clean all +RUN microdnf install -y python3.12-devel python3.12-pip shadow-utils gcc openssl-devel && microdnf clean all RUN useradd -d ${HOME_DIR} -u ${UID} -g 0 -m -s /bin/bash ${USER} \ && chown ${USER}:0 ${HOME_DIR} \ && chmod -R g+rwx ${HOME_DIR} \ From 244f16769c4b082f09333ac0e79d56ddc892b3ba Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 9 Jun 2025 20:46:29 +0800 Subject: [PATCH 27/37] RADAS: add default ignore patterns for signing --- charon/cmd/cmd_sign.py | 6 ++++-- charon/constants.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index f17fe028..9c901493 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -18,6 +18,7 @@ from charon.config import get_config from charon.pkgs.radas_sign import sign_in_radas from charon.cmd.internal import _decide_mode +from charon.constants import DEFAULT_RADAS_SIGN_IGNORES from click import command, option, argument @@ -98,8 +99,7 @@ def sign( ignore_patterns: List[str] = None, config: str = None, debug=False, - quiet=False, - dryrun=False + quiet=False ): """Do signing against files in the repo zip in repo_url through radas service. The repo_url points to the maven zip repository @@ -119,8 +119,10 @@ def sign( sys.exit(1) # All ignore files in global config should also be ignored in signing. ig_patterns = conf.get_ignore_patterns() + ig_patterns.extend(DEFAULT_RADAS_SIGN_IGNORES) if ignore_patterns: ig_patterns.extend(ignore_patterns) + ig_patterns = list(set(ig_patterns)) args = { "repo_url": repo_url, "requester": requester, diff --git a/charon/constants.py b/charon/constants.py index 35ea560a..e8056dc0 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -177,3 +177,8 @@ DEFAULT_REGISTRY = "localhost" DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 + +DEFAULT_RADAS_SIGN_IGNORES = [ + r".*\.md5$", r".*\.sha1$", r".*\.sha128$", r".*\.sha256$", + r".*\.sha512$", r".*\.asc$" +] From 925fd4941981dbe428a204fb08b938c53fc7cacc Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 11 Jun 2025 10:33:03 +0800 Subject: [PATCH 28/37] Update project version to 1.4.0 --- charon.spec | 2 +- pyproject.toml | 4 ++++ setup.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/charon.spec b/charon.spec index 95eb21a9..7fd84c7f 100644 --- a/charon.spec +++ b/charon.spec @@ -1,7 +1,7 @@ %global owner Commonjava %global modulename charon -%global charon_version 1.3.4 +%global charon_version 1.4.0 %global sdist_tar_name %{modulename}-%{charon_version} %global python3_pkgversion 3 diff --git a/pyproject.toml b/pyproject.toml index c5081bbb..6ee5f3d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,11 @@ requires = ["setuptools", "setuptools-scm"] [project] name = "charon" +<<<<<<< HEAD version = "1.3.4" +======= +version = "1.4.0" +>>>>>>> 821599b (Update project version to 1.4.0) authors = [ {name = "RedHat EXD SPMM"}, ] diff --git a/setup.py b/setup.py index 29b6833e..ee5eac4b 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ """ from setuptools import setup, find_packages -version = "1.3.4" +version = "1.4.0" long_description = """ This charon is a tool to synchronize several types of From 6fc03c2435b84e56748ffbcd99cb61ad4302a99a Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 11 Jun 2025 20:54:10 +0800 Subject: [PATCH 29/37] RADAS: use result file directly instead of folder in maven upload --- charon/cmd/cmd_upload.py | 4 ++-- charon/pkgs/maven.py | 6 +++--- charon/pkgs/radas_sign.py | 22 ++++------------------ tests/test_radas_sign_generation.py | 21 ++++----------------- 4 files changed, 13 insertions(+), 40 deletions(-) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index d56a644d..2ad294f5 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -161,7 +161,7 @@ def upload( debug=False, quiet=False, dryrun=False, - sign_result_loc="/tmp/sign" + sign_result_file=None, ): """Upload all files from a released product REPO to Ronda Service. The REPO points to a product released tarball which @@ -233,7 +233,7 @@ def upload( dry_run=dryrun, manifest_bucket_name=manifest_bucket_name, config=config, - sign_result_loc=sign_result_loc + sign_result_file=sign_result_file ) if not succeeded: sys.exit(1) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 2f525ce3..5ccee694 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -276,7 +276,7 @@ def handle_maven_uploading( dry_run=False, manifest_bucket_name=None, config=None, - sign_result_loc="/tmp/sign" + sign_result_file=None ) -> Tuple[str, bool]: """ Handle the maven product release tarball uploading process. * repo is the location of the tarball in filesystem @@ -416,10 +416,10 @@ def handle_maven_uploading( if not conf: sys.exit(1) - if conf.is_radas_enabled(): + if conf.is_radas_enabled() and sign_result_file and os.path.isfile(sign_result_file): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( - top_level=top_level, sign_result_loc=sign_result_loc + top_level=top_level, sign_result_file=sign_result_file ) if not _generated_signs: logger.error( diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 7ac4e3dc..cd229425 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -276,31 +276,17 @@ def _handle_failed_delivery(self, reason: str): self.close() -def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: +def generate_radas_sign(top_level: str, sign_result_file: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file """ - if not os.path.isdir(sign_result_loc): - logger.error("Sign result loc dir does not exist: %s", sign_result_loc) - return [], [] - - files = [ - os.path.join(sign_result_loc, f) - for f in os.listdir(sign_result_loc) - if os.path.isfile(os.path.join(sign_result_loc, f)) - ] - - if not files: - return [], [] - - if len(files) > 1: - logger.error("Multiple files found in %s. Expected only one file.", sign_result_loc) + if not sign_result_file or not os.path.isfile(sign_result_file): + logger.error("Sign result file does not exist: %s", sign_result_file) return [], [] # should only have the single sign result json file from the radas registry - json_file_path = files[0] try: - with open(json_file_path, "r") as f: + with open(sign_result_file, "r") as f: data = json.load(f) except Exception as e: logger.error("Failed to read or parse the JSON file: %s", e) diff --git a/tests/test_radas_sign_generation.py b/tests/test_radas_sign_generation.py index 33c3d695..ccc448a2 100644 --- a/tests/test_radas_sign_generation.py +++ b/tests/test_radas_sign_generation.py @@ -39,7 +39,7 @@ def tearDown(self) -> None: def test_multi_sign_files_generation(self): self.__prepare_artifacts() - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(failed, []) expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") @@ -55,7 +55,7 @@ def test_multi_sign_files_generation(self): self.assertIn("signature2@hash", content2) def test_sign_files_generation_with_missing_artifacts(self): - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(failed, []) expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") @@ -82,7 +82,7 @@ def side_effect(path, *args, **kwargs): raise IOError("mock write error") return real_open(path, *args, **kwargs) mock_open.side_effect = side_effect - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(len(failed), 1) self.assertNotIn(expected_asc1, generated) @@ -93,20 +93,7 @@ def test_sign_files_generation_with_missing_result(self): # simulate missing pull result by removing the sign result file loc shutil.rmtree(self.__sign_result_loc) - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) - self.assertEqual(failed, []) - expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") - expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") - self.assertEqual(generated, []) - self.assertFalse(os.path.exists(expected_asc1)) - self.assertFalse(os.path.exists(expected_asc2)) - - def test_sign_files_generation_with_not_single_results(self): - self.__prepare_artifacts() - another_result_file = os.path.join(self.__sign_result_loc, "result2.json") - overwrite_file(another_result_file, "test_json") - - failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_loc) + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) self.assertEqual(failed, []) expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") From 6ab9749f68fda7ebcab03ad7af1599ff9fd7a664 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 12 Jun 2025 14:00:45 +0800 Subject: [PATCH 30/37] RADAS: Added some log --- charon/cmd/cmd_sign.py | 1 - charon/pkgs/radas_sign.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py index 9c901493..49b2bf22 100644 --- a/charon/cmd/cmd_sign.py +++ b/charon/cmd/cmd_sign.py @@ -131,7 +131,6 @@ def sign( "ignore_patterns": ig_patterns, "radas_config": radas_conf } - logger.debug("params: %s", args) sign_in_radas(**args) # type: ignore except Exception: print(traceback.format_exc()) diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index cd229425..5cd87fd1 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -201,10 +201,12 @@ def __init__(self, payload: Any, rconf: RadasConfig): def on_start(self, event): self._container = event.container + self.log.debug("Start creating connection for sender") conn = self._container.connect( url=self.rconf.umb_target(), ssl_domain=self._ssl ) + self.log.debug("Connection to %s is created.", conn.hostname) if conn: self._sender = self._container.create_sender(conn, self.rconf.request_channel()) From 5c5c4db92f6affb586a93019bb236ec187fb9d27 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 12 Jun 2025 15:12:01 +0800 Subject: [PATCH 31/37] RADAS: adjust some logging --- charon/config.py | 5 ++++- charon/pkgs/radas_sign.py | 24 +++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/charon/config.py b/charon/config.py index d98bb995..65f23020 100644 --- a/charon/config.py +++ b/charon/config.py @@ -76,7 +76,10 @@ def validate(self) -> bool: return True def umb_target(self) -> str: - return f"amqps://{self.__umb_host.strip()}:{self.__umb_host_port}" + if self.ssl_enabled(): + return f"amqps://{self.__umb_host.strip()}:{self.__umb_host_port}" + else: + return f"amqp://{self.__umb_host.strip()}:{self.__umb_host_port}" def result_queue(self) -> str: return self.__result_queue.strip() diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 5cd87fd1..3ca817cd 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -201,19 +201,25 @@ def __init__(self, payload: Any, rconf: RadasConfig): def on_start(self, event): self._container = event.container - self.log.debug("Start creating connection for sender") + self.log.debug("Start creating connection for sender to %s", self.rconf.umb_target()) conn = self._container.connect( url=self.rconf.umb_target(), - ssl_domain=self._ssl + ssl_domain=self._ssl, + heartbeat=500 ) - self.log.debug("Connection to %s is created.", conn.hostname) if conn: + self.log.debug("Start creating sender") self._sender = self._container.create_sender(conn, self.rconf.request_channel()) + self.log.debug("Sender created. Remote address: %s", self._sender.target.address) + + def on_connection_opened(self, event): + conn = event.connection + self.log.debug("Connection to %s is created.", conn.hostname) def on_sendable(self, event): if not self._message_sent: msg = Message(body=self.payload, durable=True) - self.log.debug("Sending message: %s to %s", msg.id, event.sender.target.address) + self.log.debug("Sending message: %s to %s", msg.body, event.sender.target.address) self._send_msg(msg) self._message = msg self._message_sent = True @@ -232,7 +238,7 @@ def on_released(self, event): self._handle_failed_delivery("Released") def on_accepted(self, event): - self.log.info("Message accepted by receiver: %s", event.delivery) + self.log.info("Message accepted by receiver: %s", event.delivery.link.target.address) self.status = "success" self.close() # Close connection after confirmation @@ -251,26 +257,26 @@ def close(self): def _send_msg(self, msg: Message): if self._sender and self._sender.credit > 0: self._sender.send(msg) - self.log.debug("Message %s sent", msg.id) + self.log.debug("Message %s sent", msg.body) else: self.log.warning("Sender not ready or no credit available") def _handle_failed_delivery(self, reason: str): if self._pending: msg = self._pending - self.log.warning("Message %s failed for reason: %s", msg.id, reason) + self.log.warning("Message %s failed for reason: %s", msg.body, reason) max_retries = self.rconf.radas_sign_timeout_retry_count() if self._retried < max_retries: # Schedule retry self._retried = self._retried + 1 self.log.info("Scheduling retry %s/%s for message %s", - self._retried, max_retries, msg.id) + self._retried, max_retries, msg.body) # Schedule retry after delay if self._container: self._container.schedule(self.rconf.radas_sign_timeout_retry_interval(), self) else: # Max retries exceeded - self.log.error("Message %s failed after %s retries", msg.id, max_retries) + self.log.error("Message %s failed after %s retries", msg.body, max_retries) self.status = "failed" self._pending = None else: From f69202581e7e5d4bd2d2e34f9d434361cf43850b Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 13 Jun 2025 08:15:19 +0800 Subject: [PATCH 32/37] RADAS: fix radas response format --- charon/pkgs/radas_sign.py | 16 ++++++--- tests/test_radas_sign_receiver.py | 57 +++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 3ca817cd..bf63ff64 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -130,7 +130,15 @@ def _process_message(self, msg: Any) -> None: msg: The message body received """ msg_dict = json.loads(msg) - msg_request_id = msg_dict.get("request_id") + radas_response = msg_dict.get("msg") + if not radas_response: + self.log.info( + "Message %s is not valid, ignoring", + msg_dict + ) + return + + msg_request_id = radas_response.get("request_id") if msg_request_id != self.request_id: self.log.info( "Message request_id %s does not match the request_id %s from sender, ignoring", @@ -143,10 +151,10 @@ def _process_message(self, msg: Any) -> None: self.log.info( "Start to process the sign event message, request_id %s is matched", msg_request_id ) - self.sign_result_status = msg_dict.get("signing_status") - self.sign_result_errors = msg_dict.get("errors", []) + self.sign_result_status = radas_response.get("signing_status") + self.sign_result_errors = radas_response.get("errors", []) if self.sign_result_status == "success": - result_reference_url = msg_dict.get("result_reference") + result_reference_url = radas_response.get("result_reference") if not result_reference_url: self.log.warning("Not found result_reference in message，ignore.") return diff --git a/tests/test_radas_sign_receiver.py b/tests/test_radas_sign_receiver.py index e80f0435..1090c61b 100644 --- a/tests/test_radas_sign_receiver.py +++ b/tests/test_radas_sign_receiver.py @@ -60,12 +60,19 @@ def test_radas_receiver(self): # test on_message: unmatched case test_ummatch_result = { - "request_id": "test-request-id-no-match", - "file_reference": "quay.io/example/test-repo", - "result_reference": "quay.io/example-sign/sign-repo", - "sig_keyname": "testkey", - "signing_status": "success", - "errors": [] + "i": "1", + "msg_id": "test-id", + "timestamp": time.time(), + "topic": "test-topic", + "username": "test-user", + "msg": { + "request_id": "test-request-id-no-match", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } } event.message.body = json.dumps(test_ummatch_result) r_receiver.on_message(event) @@ -79,12 +86,19 @@ def test_radas_receiver(self): # test on_message: matched case with failed status self.reset_receiver(r_receiver) test_failed_result = { - "request_id": "test-request-id", - "file_reference": "quay.io/example/test-repo", - "result_reference": "quay.io/example-sign/sign-repo", - "sig_keyname": "testkey", - "signing_status": "failed", - "errors": ["error1", "error2"] + "i": "1", + "msg_id": "test-id", + "timestamp": time.time(), + "topic": "test-topic", + "username": "test-user", + "msg": { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "failed", + "errors": ["error1", "error2"] + } } event.message.body = json.dumps(test_failed_result) r_receiver.on_message(event) @@ -98,12 +112,19 @@ def test_radas_receiver(self): # test on_message: matched case with success status self.reset_receiver(r_receiver) test_success_result = { - "request_id": "test-request-id", - "file_reference": "quay.io/example/test-repo", - "result_reference": "quay.io/example-sign/sign-repo", - "sig_keyname": "testkey", - "signing_status": "success", - "errors": [] + "i": "1", + "msg_id": "test-id", + "timestamp": time.time(), + "topic": "test-topic", + "username": "test-user", + "msg": { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } } event.message.body = json.dumps(test_success_result) r_receiver.on_message(event) From ca0f12e07ce983fc2de931491f30f1c4fad992f7 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Mon, 23 Jun 2025 15:46:31 +0800 Subject: [PATCH 33/37] control oras version under 0.2.31 to make it be compatible with python 3.9 --- pyproject.toml | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6ee5f3d8..ab8b81e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "jsonschema>=4.9.1", "urllib3>=1.25.10", "semantic-version>=2.10.0", - "oras>=0.2.31", + "oras<=0.2.31", "python-qpid-proton>=0.39.0" ] diff --git a/requirements.txt b/requirements.txt index 75bb4b60..d5b5ec75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,5 @@ subresource-integrity>=0.2 jsonschema>=4.9.1 urllib3>=1.25.10 semantic-version>=2.10.0 -oras>=0.2.31 +oras<=0.2.31 python-qpid-proton>=0.39.0 \ No newline at end of file diff --git a/setup.py b/setup.py index ee5eac4b..3935d97b 100755 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ "jsonschema>=4.9.1", "urllib3>=1.25.10", "semantic-version>=2.10.0", - "oras>=0.2.31", + "oras<=0.2.31", "python-qpid-proton>=0.39.0" ], ) From a2b61ae53c8a8fcec3c6bb8fdcbb5c99b84e34b8 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 25 Jun 2025 16:49:15 +0800 Subject: [PATCH 34/37] RADAS: fix a flag typo in upload for radas signing --- charon/cmd/cmd_upload.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index 2ad294f5..3a0e6990 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -137,13 +137,11 @@ ) @option("--dryrun", "-n", is_flag=True, default=False) @option( - "--sign_result_loc", + "--sign_result_file", "-l", - default="/tmp/sign", help=""" - The local save path for oras to pull the radas signature result. - Sign request will use this path to download the signature result, - Upload will use the file on this path to generate the corresponding .asc files + The path of the file which contains radas signature result. + Upload will use the file to generate the corresponding .asc files """, ) @command() From 0498e2a5dd2085070f29c695b211b0bfe9d33081 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 26 Jun 2025 17:57:48 +0800 Subject: [PATCH 35/37] Chore: RADAS: some logging adjustment --- charon/pkgs/radas_sign.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index bf63ff64..5f4e064e 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -24,6 +24,7 @@ from typing import List, Any, Tuple, Callable, Dict, Optional from charon.config import RadasConfig from charon.pkgs.oras_client import OrasClient +from charon.utils import files from proton import SSLDomain, Message, Event, Sender, Connection from proton.handlers import MessagingHandler from proton.reactor import Container @@ -327,20 +328,25 @@ async def generate_single_sign_file( signature_path = os.path.join(top_level, asc_filename) if not os.path.isfile(artifact_path): - logger.warning("Artifact missing, skip signature file generation") + logger.warning( + "Artifact %s missing, skip signature file generation.", + artifact_path) return try: - with open(signature_path, "w") as asc_file: - asc_file.write(signature) + files.overwrite_file(signature_path, signature) generated_signs.append(signature_path) - logger.info("Generated .asc file: %s", signature_path) + logger.debug("Generated .asc file: %s", signature_path) except Exception as e: failed_paths.append(signature_path) logger.error("Failed to write .asc file for %s: %s", artifact_path, e) result = data.get("results", []) - return __do_path_cut_and(path_handler=generate_single_sign_file, data=result) + (_failed_metas, _generated_signs) = __do_path_cut_and(generate_single_sign_file, result) + logger.info( + "Signature generation done. There are %s signature files generated.", + len(_generated_signs)) + return (_failed_metas, _generated_signs) def __do_path_cut_and( From 8a7774b6cbd1a2cc3488960a473e03045518efa0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 26 Jun 2025 18:48:32 +0800 Subject: [PATCH 36/37] Fix: RADAS: fix a list index out of bounds issue --- charon/pkgs/radas_sign.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py index 5f4e064e..fcdf6e49 100644 --- a/charon/pkgs/radas_sign.py +++ b/charon/pkgs/radas_sign.py @@ -168,7 +168,8 @@ def _process_message(self, msg: Any) -> None: files = oras_client.pull( result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc ) - self.log.info("Number of files pulled: %d, path: %s", len(files), files[0]) + if files and len(files) > 0: + self.log.info("Number of files pulled: %d, path: %s", len(files), files[0]) else: self.log.error("The signing result received with failed status. Errors: %s", self.sign_result_errors) From ed636d8c549dbda266938af8dbf82617743048b0 Mon Sep 17 00:00:00 2001 From: Gang Li Date: Fri, 27 Jun 2025 15:57:02 +0800 Subject: [PATCH 37/37] Update charon.spec and pyproject.toml with version update --- charon.spec | 17 +++++++++++++++++ pyproject.toml | 4 ---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/charon.spec b/charon.spec index 7fd84c7f..fb9f2073 100644 --- a/charon.spec +++ b/charon.spec @@ -64,6 +64,23 @@ export LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8 %changelog +* Fri Jun 27 2025 Gang Li +- 1.4.0 release +- Add RADAS signature support + +* Mon Jun 23 2025 Gang Li +- 1.3.4 release +- Fix the sorting problem of index page items + +* Mon Dec 16 2024 Gang Li +- 1.3.3 release +- Fix npm del error when deleting a package which has overlapped name with others +- Some code refinement + +* Thu Jul 11 2024 Gang Li +- 1.3.2 release +- Some updates in the Containerfile. + * Tue May 7 2024 Gang Li - 1.3.1 release - Add checksum refresh command: refresh checksum files for maven artifacts diff --git a/pyproject.toml b/pyproject.toml index ab8b81e7..b667868d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,11 +4,7 @@ requires = ["setuptools", "setuptools-scm"] [project] name = "charon" -<<<<<<< HEAD -version = "1.3.4" -======= version = "1.4.0" ->>>>>>> 821599b (Update project version to 1.4.0) authors = [ {name = "RedHat EXD SPMM"}, ]