From 846dd447d9db62c6e98e4662699e16a384081cbd Mon Sep 17 00:00:00 2001 From: dbasunag Date: Thu, 30 May 2024 18:03:58 -0400 Subject: [PATCH 01/17] jira initial pr This reverts commit 017044d308a264b11b140d286d157a4edfb6fc2c. --- apps/jira_utils/__init__.py | 0 apps/jira_utils/get_jira_information.py | 40 +++++++ apps/jira_utils/utils.py | 137 +++++++++++++++++++++++ apps/utils.py | 2 +- example.jira.cfg | 4 + poetry.lock | 139 ++++++++++++++++++++++++ pyproject.toml | 6 + tests/jira_utils/__init__.py | 0 tests/jira_utils/test_jira.py | 34 ++++++ 9 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 apps/jira_utils/__init__.py create mode 100644 apps/jira_utils/get_jira_information.py create mode 100644 apps/jira_utils/utils.py create mode 100644 example.jira.cfg create mode 100644 tests/jira_utils/__init__.py create mode 100644 tests/jira_utils/test_jira.py diff --git a/apps/jira_utils/__init__.py b/apps/jira_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py new file mode 100644 index 0000000..bd79713 --- /dev/null +++ b/apps/jira_utils/get_jira_information.py @@ -0,0 +1,40 @@ +import os +import click + +from simple_logger.logger import get_logger + +from apps.jira_utils.utils import get_jiras_from_python_files, get_closed_jira_ids, get_jira_connection_params, \ + get_jira_version_mismatch, JiraConnector +from apps.utils import ListParamType + +LOGGER = get_logger(name=__name__) + + +@click.command() +@click.option( + "--jira-cfg-file", + help="Provide absolute path to the jira_utils config file. ", + type=click.Path(), + default=os.path.expanduser("~/.config/python-utility-scripts/jira_utils/config.cfg"), +) +@click.option( + "--jira-target-versions", + help="Provide comma separated list of Jira target version for the bugs.", + type=ListParamType(), + required=True +) +def get_closed_jiras(jira_cfg_file, jira_target_versions): + jira_connector = JiraConnector(cfg_file=jira_cfg_file) + jira_id_dict = get_jiras_from_python_files() + if jira_errors:= get_closed_jira_ids(jira_connector=jira_connector, + jira_ids_dict=jira_id_dict): + click.echo(f"Following jiras are not open or could not be accessed: {jira_errors}") + raise click.Abort() + if version_mismatch := get_jira_version_mismatch(jira_connector=jira_connector, jira_id_dict=jira_id_dict, + jira_expected_versions=jira_target_versions): + click.echo(f"Following jiras are not matching expected version{jira_target_versions}: {version_mismatch}") + raise click.Abort() + + +if __name__ == "__main__": + get_closed_jiras() \ No newline at end of file diff --git a/apps/jira_utils/utils.py b/apps/jira_utils/utils.py new file mode 100644 index 0000000..ec32ad9 --- /dev/null +++ b/apps/jira_utils/utils.py @@ -0,0 +1,137 @@ +import os +import re +import click +from configparser import ConfigParser +from packaging.version import InvalidVersion, Version +from simple_logger.logger import get_logger +from jira import JIRA, JIRAError + +from apps.utils import all_python_files + +LOGGER = get_logger(name=__name__) + + +def get_jira_connection_params(conf_file_name): + if os.path.exists(conf_file_name): + parser = ConfigParser() + parser.read(conf_file_name, encoding="utf-8") + params_dict = {} + for params in parser.items("DEFAULT"): + params_dict[params[0]] = params[1] + return params_dict + click.echo("Jira config file is required.") + raise click.Abort() + + +class JiraConnector: + def __init__(self, cfg_file): + self.cfg_file = cfg_file + config_dict = get_jira_connection_params(conf_file_name=self.cfg_file) + self.token = config_dict["token"] + self.server = config_dict["url"] + if not (self.token and self.server): + raise ValueError("Jira config file must contain token and server information.") + + self.resolved_statuses = config_dict.get('resolved_statuses') + self.project_ids = config_dict.get('project_ids') + self.jira = None + self.authenticate_to_jira_server() + + def authenticate_to_jira_server(self): + try: + self.jira = JIRA(token_auth=self.token, options={"server": self.server}) + LOGGER.info("Connected to Jira") + except JIRAError as e: + LOGGER.error("Failed to connect to Jira: %s", e) + raise + + def get_issue_metadata(self, jira_id, max_retry=3): + retries = 0 + while True: + try: + return self.jira.issue(id=jira_id, fields="status, issuetype, fixVersions").fields + except JIRAError as e: + # Check for inactivity error (adjust based on your library) + if "401 Unauthorized" in str(e) or "Session timed out" in str(e): + retries += 1 + LOGGER.warning("Failed to get issue due to inactivity, retrying (%d/%d)", retries, max_retry) + if retries < max_retry: + self.authenticate_to_jira_server() # Attempt reconnection + else: + raise # Re-raise the error after exceeding retries + else: + raise + + +def get_all_jiras_from_file(file_content): + """ + Try to find all jira_utils tickets in the file. + Looking for the following patterns: + - jira_id=CNV-12345 # call in is_jira_open + - https://issues.redhat.com/browse/CNV-12345 # when jira_utils is in a link in comments + - pytest.mark.jira_utils(CNV-12345) # when jira_utils is in a marker + + Args: + file_content (str): The content of the file. + + Returns: + list: A list of jira_utils tickets. + """ + issue_pattern = r"([A-Z]+-[0-9]+)" + _pytest_jira_marker_bugs = re.findall( + rf"pytest.mark.jira.*?{issue_pattern}.*", file_content, re.DOTALL + ) + _is_jira_open = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content) + _jira_url_jiras = re.findall( + rf"https://issues.redhat.com/browse/{issue_pattern}(?! )", + file_content, + ) + return list(set(_pytest_jira_marker_bugs + _is_jira_open + _jira_url_jiras)) + + +def get_jiras_from_python_files(): + jira_found = {} + for filename in all_python_files(): + with open(filename) as fd: + if unique_jiras := get_all_jiras_from_file(file_content=fd.read()): + jira_found[filename] = unique_jiras + LOGGER.warning(f"File: {filename}, {unique_jiras}") + return jira_found + + +def get_closed_jira_ids(jira_connector, jira_ids_dict): + jira_errors = {} + for file_name in jira_ids_dict: + for jira_id in jira_ids_dict[file_name]: + try: + current_jira_status = jira_connector.get_issue_metadata(jira_id=jira_id).status.name.lower() + if current_jira_status in jira_connector.resolved_statuses: + jira_errors.setdefault(file_name, []).append( + f"{jira_id} [{current_jira_status}]" + ) + except JIRAError as exp: + jira_errors.setdefault(file_name, []).append( + f"{jira_id} [{exp.text}]" + ) + continue + return jira_errors + + +def get_jira_version_mismatch(jira_connector, jira_id_dict, jira_expected_versions=None): + jira_mismatch = {} + for file_name in jira_id_dict: + unique_jira_ids = jira_id_dict[file_name] + if jira_connector.project_ids: + unique_jira_ids = [jira_id for jira_id in unique_jira_ids + if jira_id.startswith(tuple(jira_connector.project_ids))] + for jira_id in unique_jira_ids: + jira_issue = jira_connector.get_issue_metadata(jira_id=jira_id) + fix_version = re.search(r"([\d.]+)", jira_issue.fixVersions[0].name) if jira_issue.fixVersions else None + jira_target_release_version = fix_version.group(1) if fix_version else "vfuture" + LOGGER.info(f"issue: {jira_id}, version: {jira_target_release_version}, {jira_expected_versions}") + if not jira_target_release_version.startswith(tuple(jira_expected_versions)): + jira_mismatch.setdefault(file_name, []).append( + f"{jira_id} [{jira_target_release_version}]" + ) + + return jira_mismatch \ No newline at end of file diff --git a/apps/utils.py b/apps/utils.py index 3b1add6..b8fe356 100644 --- a/apps/utils.py +++ b/apps/utils.py @@ -80,7 +80,7 @@ def all_python_files() -> Iterable[str]: for root, _, files in os.walk(os.path.abspath(os.curdir)): if [_dir for _dir in exclude_dirs if _dir in root]: continue - + LOGGER.info(root) for filename in files: if filename.endswith(".py"): yield os.path.join(root, filename) diff --git a/example.jira.cfg b/example.jira.cfg new file mode 100644 index 0000000..67ce736 --- /dev/null +++ b/example.jira.cfg @@ -0,0 +1,4 @@ +[DEFAULT] +url = https://www.atlassian.com/ +token = mytoken +resolved_statuses = (verified, release pending, closed) \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 2d0e74a..65263fe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -428,6 +428,17 @@ files = [ {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +description = "XML bomb protection for Python stdlib modules" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, + {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, +] + [[package]] name = "exceptiongroup" version = "1.2.0" @@ -576,6 +587,34 @@ profiling = ["gprof2dot"] rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "jira" +version = "3.6.0" +description = "Python library for interacting with JIRA via REST APIs." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jira-3.6.0-py3-none-any.whl", hash = "sha256:08b28388ee498542ebb6b05db87e6c46c37535c268717ccc23c84b377ea309fb"}, + {file = "jira-3.6.0.tar.gz", hash = "sha256:4c67497fe8dc2f60f1c4f7b33479f059c928bec3db9dcb5cd7b6a09b6ecc0942"}, +] + +[package.dependencies] +defusedxml = "*" +packaging = "*" +Pillow = ">=2.1.0" +requests = ">=2.10.0" +requests-oauthlib = ">=1.1.0" +requests-toolbelt = "*" +typing-extensions = ">=3.7.4.2" + +[package.extras] +async = ["requests-futures (>=0.9.7)"] +cli = ["ipython (>=4.0.0)", "keyring"] +docs = ["furo", "sphinx (>=5.0.0)", "sphinx-copybutton"] +opt = ["PyJWT", "filemagic (>=1.6)", "requests-jwt", "requests-kerberos"] +test = ["MarkupSafe (>=0.23)", "PyYAML (>=5.1)", "docutils (>=0.12)", "flaky", "oauthlib", "parameterized (>=0.8.1)", "pytest (>=6.0.0)", "pytest-cache", "pytest-cov", "pytest-instafail", "pytest-sugar", "pytest-timeout (>=1.3.1)", "pytest-xdist (>=2.2)", "requests-mock", "requires.io", "tenacity", "wheel (>=0.24.0)", "yanc (>=0.3.3)"] + [[package]] name = "matplotlib-inline" version = "0.1.6" @@ -698,6 +737,91 @@ files = [ {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, ] +[[package]] +name = "pillow" +version = "10.2.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pillow-10.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:7823bdd049099efa16e4246bdf15e5a13dbb18a51b68fa06d6c1d4d8b99a796e"}, + {file = "pillow-10.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:83b2021f2ade7d1ed556bc50a399127d7fb245e725aa0113ebd05cfe88aaf588"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fad5ff2f13d69b7e74ce5b4ecd12cc0ec530fcee76356cac6742785ff71c452"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da2b52b37dad6d9ec64e653637a096905b258d2fc2b984c41ae7d08b938a67e4"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:47c0995fc4e7f79b5cfcab1fc437ff2890b770440f7696a3ba065ee0fd496563"}, + {file = "pillow-10.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:322bdf3c9b556e9ffb18f93462e5f749d3444ce081290352c6070d014c93feb2"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:51f1a1bffc50e2e9492e87d8e09a17c5eea8409cda8d3f277eb6edc82813c17c"}, + {file = "pillow-10.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:69ffdd6120a4737710a9eee73e1d2e37db89b620f702754b8f6e62594471dee0"}, + {file = "pillow-10.2.0-cp310-cp310-win32.whl", hash = "sha256:c6dafac9e0f2b3c78df97e79af707cdc5ef8e88208d686a4847bab8266870023"}, + {file = "pillow-10.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:aebb6044806f2e16ecc07b2a2637ee1ef67a11840a66752751714a0d924adf72"}, + {file = "pillow-10.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:7049e301399273a0136ff39b84c3678e314f2158f50f517bc50285fb5ec847ad"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:35bb52c37f256f662abdfa49d2dfa6ce5d93281d323a9af377a120e89a9eafb5"}, + {file = "pillow-10.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c23f307202661071d94b5e384e1e1dc7dfb972a28a2310e4ee16103e66ddb67"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:773efe0603db30c281521a7c0214cad7836c03b8ccff897beae9b47c0b657d61"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11fa2e5984b949b0dd6d7a94d967743d87c577ff0b83392f17cb3990d0d2fd6e"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:716d30ed977be8b37d3ef185fecb9e5a1d62d110dfbdcd1e2a122ab46fddb03f"}, + {file = "pillow-10.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a086c2af425c5f62a65e12fbf385f7c9fcb8f107d0849dba5839461a129cf311"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c8de2789052ed501dd829e9cae8d3dcce7acb4777ea4a479c14521c942d395b1"}, + {file = "pillow-10.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:609448742444d9290fd687940ac0b57fb35e6fd92bdb65386e08e99af60bf757"}, + {file = "pillow-10.2.0-cp311-cp311-win32.whl", hash = "sha256:823ef7a27cf86df6597fa0671066c1b596f69eba53efa3d1e1cb8b30f3533068"}, + {file = "pillow-10.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:1da3b2703afd040cf65ec97efea81cfba59cdbed9c11d8efc5ab09df9509fc56"}, + {file = "pillow-10.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:edca80cbfb2b68d7b56930b84a0e45ae1694aeba0541f798e908a49d66b837f1"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:1b5e1b74d1bd1b78bc3477528919414874748dd363e6272efd5abf7654e68bef"}, + {file = "pillow-10.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0eae2073305f451d8ecacb5474997c08569fb4eb4ac231ffa4ad7d342fdc25ac"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7c2286c23cd350b80d2fc9d424fc797575fb16f854b831d16fd47ceec078f2c"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e23412b5c41e58cec602f1135c57dfcf15482013ce6e5f093a86db69646a5aa"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:52a50aa3fb3acb9cf7213573ef55d31d6eca37f5709c69e6858fe3bc04a5c2a2"}, + {file = "pillow-10.2.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:127cee571038f252a552760076407f9cff79761c3d436a12af6000cd182a9d04"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8d12251f02d69d8310b046e82572ed486685c38f02176bd08baf216746eb947f"}, + {file = "pillow-10.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:54f1852cd531aa981bc0965b7d609f5f6cc8ce8c41b1139f6ed6b3c54ab82bfb"}, + {file = "pillow-10.2.0-cp312-cp312-win32.whl", hash = "sha256:257d8788df5ca62c980314053197f4d46eefedf4e6175bc9412f14412ec4ea2f"}, + {file = "pillow-10.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:154e939c5f0053a383de4fd3d3da48d9427a7e985f58af8e94d0b3c9fcfcf4f9"}, + {file = "pillow-10.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:f379abd2f1e3dddb2b61bc67977a6b5a0a3f7485538bcc6f39ec76163891ee48"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8373c6c251f7ef8bda6675dd6d2b3a0fcc31edf1201266b5cf608b62a37407f9"}, + {file = "pillow-10.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:870ea1ada0899fd0b79643990809323b389d4d1d46c192f97342eeb6ee0b8483"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b6b1e20608493548b1f32bce8cca185bf0480983890403d3b8753e44077129"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3031709084b6e7852d00479fd1d310b07d0ba82765f973b543c8af5061cf990e"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:3ff074fc97dd4e80543a3e91f69d58889baf2002b6be64347ea8cf5533188213"}, + {file = "pillow-10.2.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:cb4c38abeef13c61d6916f264d4845fab99d7b711be96c326b84df9e3e0ff62d"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b1b3020d90c2d8e1dae29cf3ce54f8094f7938460fb5ce8bc5c01450b01fbaf6"}, + {file = "pillow-10.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:170aeb00224ab3dc54230c797f8404507240dd868cf52066f66a41b33169bdbe"}, + {file = "pillow-10.2.0-cp38-cp38-win32.whl", hash = "sha256:c4225f5220f46b2fde568c74fca27ae9771536c2e29d7c04f4fb62c83275ac4e"}, + {file = "pillow-10.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0689b5a8c5288bc0504d9fcee48f61a6a586b9b98514d7d29b840143d6734f39"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:b792a349405fbc0163190fde0dc7b3fef3c9268292586cf5645598b48e63dc67"}, + {file = "pillow-10.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c570f24be1e468e3f0ce7ef56a89a60f0e05b30a3669a459e419c6eac2c35364"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8ecd059fdaf60c1963c58ceb8997b32e9dc1b911f5da5307aab614f1ce5c2fb"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c365fd1703040de1ec284b176d6af5abe21b427cb3a5ff68e0759e1e313a5e7e"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:70c61d4c475835a19b3a5aa42492409878bbca7438554a1f89d20d58a7c75c01"}, + {file = "pillow-10.2.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b6f491cdf80ae540738859d9766783e3b3c8e5bd37f5dfa0b76abdecc5081f13"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d189550615b4948f45252d7f005e53c2040cea1af5b60d6f79491a6e147eef7"}, + {file = "pillow-10.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:49d9ba1ed0ef3e061088cd1e7538a0759aab559e2e0a80a36f9fd9d8c0c21591"}, + {file = "pillow-10.2.0-cp39-cp39-win32.whl", hash = "sha256:babf5acfede515f176833ed6028754cbcd0d206f7f614ea3447d67c33be12516"}, + {file = "pillow-10.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:0304004f8067386b477d20a518b50f3fa658a28d44e4116970abfcd94fac34a8"}, + {file = "pillow-10.2.0-cp39-cp39-win_arm64.whl", hash = "sha256:0fb3e7fc88a14eacd303e90481ad983fd5b69c761e9e6ef94c983f91025da869"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:322209c642aabdd6207517e9739c704dc9f9db943015535783239022002f054a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eedd52442c0a5ff4f887fab0c1c0bb164d8635b32c894bc1faf4c618dd89df2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb28c753fd5eb3dd859b4ee95de66cc62af91bcff5db5f2571d32a520baf1f04"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:33870dc4653c5017bf4c8873e5488d8f8d5f8935e2f1fb9a2208c47cdd66efd2"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3c31822339516fb3c82d03f30e22b1d038da87ef27b6a78c9549888f8ceda39a"}, + {file = "pillow-10.2.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a2b56ba36e05f973d450582fb015594aaa78834fefe8dfb8fcd79b93e64ba4c6"}, + {file = "pillow-10.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d8e6aeb9201e655354b3ad049cb77d19813ad4ece0df1249d3c793de3774f8c7"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:2247178effb34a77c11c0e8ac355c7a741ceca0a732b27bf11e747bbc950722f"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15587643b9e5eb26c48e49a7b33659790d28f190fc514a322d55da2fb5c2950e"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753cd8f2086b2b80180d9b3010dd4ed147efc167c90d3bf593fe2af21265e5a5"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7c8f97e8e7a9009bcacbe3766a36175056c12f9a44e6e6f2d5caad06dcfbf03b"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d1b35bcd6c5543b9cb547dee3150c93008f8dd0f1fef78fc0cd2b141c5baf58a"}, + {file = "pillow-10.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe4c15f6c9285dc54ce6553a3ce908ed37c8f3825b5a51a15c91442bb955b868"}, + {file = "pillow-10.2.0.tar.gz", hash = "sha256:e87f0b2c78157e12d7686b27d63c070fd65d994e8ddae6f328e0dcf4a0cd007e"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +typing = ["typing-extensions"] +xmp = ["defusedxml"] + [[package]] name = "pluggy" version = "1.5.0" @@ -1022,6 +1146,21 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9 [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +description = "A utility belt for advanced users of python-requests" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, + {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, +] + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" [[package]] name = "six" diff --git a/pyproject.toml b/pyproject.toml index 0275410..a0fbc71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,12 @@ packages = [{ include = "apps" }] [tool.poetry.scripts] pyutils-unusedcode = "apps.unused_code.unused_code:get_unused_functions" pyutils-polarion-verify-tc-requirements = "apps.polarion.polarion_verify_tc_requirements:has_verify" +<<<<<<< Updated upstream pyutils-polarion-set-automated = "apps.polarion.polarion_set_automated:polarion_approve_automate" +======= +pyutils-jira = "apps.jira_utils.get_jira_information:get_closed_jiras" + +>>>>>>> Stashed changes [tool.poetry.dependencies] python = "^3.8" @@ -56,6 +61,7 @@ pylero = "^0.1.0" pyhelper-utils = "^0.0.41" pytest-mock = "^3.14.0" pyyaml = "^6.0.1" +jira = "^3.6.0" [tool.poetry.group.dev.dependencies] ipdb = "^0.13.13" diff --git a/tests/jira_utils/__init__.py b/tests/jira_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/jira_utils/test_jira.py b/tests/jira_utils/test_jira.py new file mode 100644 index 0000000..553f7c5 --- /dev/null +++ b/tests/jira_utils/test_jira.py @@ -0,0 +1,34 @@ +from apps.jira_utils.get_jira_information import get_closed_jiras +from tests.utils import get_cli_runner +from simple_logger.logger import get_logger +from unittest import mock + +LOGGER = get_logger(name=__name__) + + +def test_jira_missing_target_version(): + result = get_cli_runner().invoke(get_closed_jiras, "--jira-cfg-file example.jira.cfg") + LOGGER.info(f"Result output: {result.output}, exit code: {result.exit_code}, exceptions: {result.exception}") + assert result.exit_code != 0 + assert "Missing option '--jira-target-versions'" in result.output + + +def test_jira_missing_cfg_file(): + result = get_cli_runner().invoke(get_closed_jiras, '--jira-cfg-file jira.cfg --jira-target-versions "ABC"') + LOGGER.info(f"Result output: {result.output}, exit code: {result.exit_code}, exceptions: {result.exception}") + assert result.exit_code == 1 + assert "Jira config file is required." in result.output + + +def test_jira_with_closed_bugs(): + with mock.patch("apps.jira_utils.get_jira_information.JiraConnector") as jira_connector: + a_instance = jira_connector.return_value + a_instance.get_issue_metadata.return_value = {'status':{'name':'closed'}} + with mock.patch("apps.jira_utils.get_jira_information.get_jiras_from_python_files") as jiras: + jiras.return_value = {"file1": ['ABC-1234']} + with mock.patch("apps.jira_utils.get_jira_information.JiraConnector.get_closed_jira_ids") as closed_ids: + closed_ids.return_value = "{'file1': ['ABC-1234' ['closed']]}" + result = get_cli_runner().invoke(get_closed_jiras, '--jira-target-versions "ABC"') + assert result.exit_code == 1 + #assert f"Following jiras are not open or could not be accessed:" in result.output + LOGGER.info(f"Result output: {result.output}, exit code: {result.exit_code}, exceptions: {result.exception}") From 2e8ca1d3e00f6c9a39c98a7675b08436c1127396 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 22:05:53 +0000 Subject: [PATCH 02/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- apps/jira_utils/get_jira_information.py | 2 +- apps/jira_utils/utils.py | 2 +- example.jira.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py index bd79713..85e052e 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/get_jira_information.py @@ -37,4 +37,4 @@ def get_closed_jiras(jira_cfg_file, jira_target_versions): if __name__ == "__main__": - get_closed_jiras() \ No newline at end of file + get_closed_jiras() diff --git a/apps/jira_utils/utils.py b/apps/jira_utils/utils.py index ec32ad9..d09b4ca 100644 --- a/apps/jira_utils/utils.py +++ b/apps/jira_utils/utils.py @@ -134,4 +134,4 @@ def get_jira_version_mismatch(jira_connector, jira_id_dict, jira_expected_versio f"{jira_id} [{jira_target_release_version}]" ) - return jira_mismatch \ No newline at end of file + return jira_mismatch diff --git a/example.jira.cfg b/example.jira.cfg index 67ce736..569fee4 100644 --- a/example.jira.cfg +++ b/example.jira.cfg @@ -1,4 +1,4 @@ [DEFAULT] url = https://www.atlassian.com/ token = mytoken -resolved_statuses = (verified, release pending, closed) \ No newline at end of file +resolved_statuses = (verified, release pending, closed) From d0090794685146811054d6b844e30e8744afc509 Mon Sep 17 00:00:00 2001 From: dbasunag Date: Fri, 2 Aug 2024 19:09:16 -0400 Subject: [PATCH 03/17] updates to jira --- apps/jira_utils/README.md | 52 +++++++++ apps/jira_utils/exceptions.py | 6 ++ apps/jira_utils/get_jira_information.py | 88 ++++++++++++--- apps/jira_utils/jira_utils.py | 64 +++++++++++ apps/jira_utils/utils.py | 137 ------------------------ apps/utils.py | 1 - example.jira.cfg | 4 - example.jira.yaml | 12 +++ poetry.lock | 117 +++++++++++++------- pyproject.toml | 6 +- tests/jira_utils/test_jira.py | 45 +++----- 11 files changed, 304 insertions(+), 228 deletions(-) create mode 100644 apps/jira_utils/README.md create mode 100644 apps/jira_utils/exceptions.py create mode 100644 apps/jira_utils/jira_utils.py delete mode 100644 apps/jira_utils/utils.py delete mode 100644 example.jira.cfg create mode 100644 example.jira.yaml diff --git a/apps/jira_utils/README.md b/apps/jira_utils/README.md new file mode 100644 index 0000000..8cc550d --- /dev/null +++ b/apps/jira_utils/README.md @@ -0,0 +1,52 @@ +# pyutils-jira +Helper to identify wrong jira card references in a repository files. e.g. closed tickets or tickets with not relevant target versions in a test repository branch would be identified. +It looks for following patterns in files of a pytest repository: +```bash + - jira_id=ABC-12345 # When jira id is present in a function call + - https://issues.redhat.com/browse/ABC-12345 # when jira is in a link in comments + - pytest.mark.jira_utils(ABC-12345) # when jira is in a marker +``` + +## Usage + +```bash +pyutils-jira +pyutils-jira --help +``` + +## Config file +A config file with the jira connection parameters like url, token, resolved_statuses, skip_project_ids, jira_target_versions should be added to +`~/.config/python-utility-scripts/jira_utils/config.cfg` + +### Example: + +```yaml +pyutils-jira: + url: + token: mytoken + resolved_statuses: + - verified + - release pending + - closed + skip_project_ids: + - ABC + - DEF + jira_target_versions: + - 1.0.0 + - 2.0.1 +``` +This would skip version checks on any jira ids associated with project ABC and DEF +This would also check if the current repository branch is pointing to any jira card that is not targeted for 1.0.0 or 2.0.1 version + +To run from CLI with `--jira-target-versions` + +```bash +pyutils-unusedcode --jira-target-versions '1.0.0,2.0.1' +``` +#### Note: +To mark to skip a jira from these checks, one can add `` as a comment to the same line containing the specific jira + +Example: +```bash + #https://issues.redhat.com/browse/OSD-5716 + ``` diff --git a/apps/jira_utils/exceptions.py b/apps/jira_utils/exceptions.py new file mode 100644 index 0000000..7521d32 --- /dev/null +++ b/apps/jira_utils/exceptions.py @@ -0,0 +1,6 @@ +class JiraInvalidConfigFileError(Exception): + pass + + +class JiraValidationError(Exception): + pass diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py index 85e052e..60186ba 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/get_jira_information.py @@ -1,11 +1,18 @@ +import logging import os +import re + import click +from jira import JIRAError from simple_logger.logger import get_logger -from apps.jira_utils.utils import get_jiras_from_python_files, get_closed_jira_ids, get_jira_connection_params, \ - get_jira_version_mismatch, JiraConnector -from apps.utils import ListParamType +from apps.jira_utils.exceptions import JiraValidationError, JiraInvalidConfigFileError +from apps.jira_utils.jira_utils import get_jiras_from_python_files, JiraConnector +from apps.utils import ListParamType, get_util_config +from typing import Any, Dict + +DEFAULT_RESOLVED_STATUS = ["verified", "release pending", "closed", "resolved"] LOGGER = get_logger(name=__name__) @@ -19,22 +26,69 @@ ) @click.option( "--jira-target-versions", - help="Provide comma separated list of Jira target version for the bugs.", + help="Provide comma separated list of Jira target version, for version validation against a repo branch.", type=ListParamType(), - required=True + required=False, ) -def get_closed_jiras(jira_cfg_file, jira_target_versions): - jira_connector = JiraConnector(cfg_file=jira_cfg_file) - jira_id_dict = get_jiras_from_python_files() - if jira_errors:= get_closed_jira_ids(jira_connector=jira_connector, - jira_ids_dict=jira_id_dict): - click.echo(f"Following jiras are not open or could not be accessed: {jira_errors}") - raise click.Abort() - if version_mismatch := get_jira_version_mismatch(jira_connector=jira_connector, jira_id_dict=jira_id_dict, - jira_expected_versions=jira_target_versions): - click.echo(f"Following jiras are not matching expected version{jira_target_versions}: {version_mismatch}") - raise click.Abort() +@click.option("--verbose", default=False, is_flag=True) +def get_jira_mismatch(jira_cfg_file: Any, jira_target_versions: Any, verbose: bool) -> Any: + if verbose: + LOGGER.setLevel(logging.DEBUG) + else: + logging.disable(logging.CRITICAL) + config_dict = get_util_config(util_name="pyutils-jira", config_file_path=jira_cfg_file) + if not (config_dict.get("url") and config_dict.get("token")): + raise JiraInvalidConfigFileError("Jira config file must contain valid url and token.") + jira_connector = JiraConnector(token=config_dict["token"], url=config_dict["url"]) + jira_error: Dict[str, Any] = {"status_mismatch": {}, "version_mismatch": {}, "connection_error": {}} + resolved_status = config_dict.get("resolved_statuses") or DEFAULT_RESOLVED_STATUS + jira_target_versions = jira_target_versions or config_dict.get("jira_target_versions", []) + skip_project_ids = config_dict.get("skip_project_ids", []) + for file_name in (jira_id_dict := get_jiras_from_python_files()): + for jira_id in jira_id_dict[file_name]: + try: + # check closed status: + jira_issue_metadata = jira_connector.get_issue(jira_id=jira_id).fields + current_jira_status = jira_issue_metadata.status.name.lower() + if current_jira_status in resolved_status: + jira_error["status_mismatch"].setdefault(file_name, []).append( + f"{jira_id}: current status: {current_jira_status}" + ) + # validate correct target version if provided: + if jira_target_versions: + if skip_project_ids and jira_id.startswith(tuple(skip_project_ids)): + continue + fix_version = ( + re.search(r"([\d.]+)", jira_issue_metadata.fixVersions[0].name) + if (jira_issue_metadata.fixVersions) + else None + ) + current_target_version = fix_version.group(1) if fix_version else "vfuture" + if not any([current_target_version == version for version in jira_target_versions]): + jira_error["version_mismatch"].setdefault(file_name, []).append( + f"{jira_id}: target version: {current_target_version}]" + ) + + except JIRAError as exp: + jira_error["connection_error"].setdefault(file_name, []).append( + f"{jira_id}: status code: {exp.status_code}, details: {exp.text}]." + ) + jira_error = {key: value for key, value in jira_error.items() if value} + if jira_error.values(): + error = "Following Jira ids failed jira check:\n" + if jira_error.get("status_mismatch"): + error += f" Jira ids in resolved state: {jira_error['status_mismatch']}." + if jira_error.get("version_mismatch"): + error += ( + f" Jira expected versions: {jira_target_versions}, " + f"current versions: {jira_error['version_mismatch']}." + ) + if jira_error.get("connection_error"): + error += f" Jira ids with connection error: {jira_error['connection_error']}." + LOGGER.error(error) + raise JiraValidationError(error) + LOGGER.info("Successfully completed Jira validations") if __name__ == "__main__": - get_closed_jiras() + get_jira_mismatch() diff --git a/apps/jira_utils/jira_utils.py b/apps/jira_utils/jira_utils.py new file mode 100644 index 0000000..23fe73c --- /dev/null +++ b/apps/jira_utils/jira_utils.py @@ -0,0 +1,64 @@ +import re +from simple_logger.logger import get_logger +from jira import JIRA, JIRAError, Issue +from retry import retry +from apps.utils import all_python_files +from typing import List, Dict, Any + +LOGGER = get_logger(name=__name__) + + +class JiraConnector: + def __init__(self, token: str, url: str) -> None: + self.token = token + self.url = url + self.jira = JIRA(token_auth=self.token, options={"server": self.url}) + + @retry(JIRAError, tries=3, delay=2) + def get_issue(self, jira_id: str) -> Issue: + return self.jira.issue(id=jira_id, fields="status, issuetype, fixVersions") + + +def get_jira_ids_from_file_content(file_content: str) -> List[str]: + """ + Try to find all jira_utils tickets in a given file content. + Looking for the following patterns: + - jira_id=ABC-12345 # When jira id is present in a function call + - https://issues.redhat.com/browse/ABC-12345 # when jira is in a link in comments + - pytest.mark.jira_utils(ABC-12345) # when jira is in a marker + + Args: + file_content (str): The content of a given file. + + Returns: + list: A list of jira tickets. + """ + issue_pattern = r"([A-Z]+-[0-9]+)" + _pytest_jira_marker_bugs = re.findall(rf"pytest.mark.jira.*?{issue_pattern}.*", file_content, re.DOTALL) + _is_jira_open = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content) + _jira_url_jiras = [] + _jira_url_jiras = re.findall( + rf"https://issues.redhat.com/browse/{issue_pattern}", + file_content, + ) + return list(set(_pytest_jira_marker_bugs + _is_jira_open + _jira_url_jiras)) + + +def get_jiras_from_python_files() -> Dict[str, Any]: + """ + Get all python files from the current directory and get list of jira ids from each of them + + Note: any line containing would be not be checked for presence of a jira id + """ + jira_found = {} + for filename in all_python_files(): + with open(filename) as fd: + file_content = [] + for line in fd.readlines(): + # if appears in a line, exclude that line from jira check + if "" not in line: + file_content.append(line) + if unique_jiras := get_jira_ids_from_file_content(file_content="\n".join(file_content)): + jira_found[filename] = unique_jiras + LOGGER.info(f"File: {filename}, {unique_jiras}") + return jira_found diff --git a/apps/jira_utils/utils.py b/apps/jira_utils/utils.py deleted file mode 100644 index d09b4ca..0000000 --- a/apps/jira_utils/utils.py +++ /dev/null @@ -1,137 +0,0 @@ -import os -import re -import click -from configparser import ConfigParser -from packaging.version import InvalidVersion, Version -from simple_logger.logger import get_logger -from jira import JIRA, JIRAError - -from apps.utils import all_python_files - -LOGGER = get_logger(name=__name__) - - -def get_jira_connection_params(conf_file_name): - if os.path.exists(conf_file_name): - parser = ConfigParser() - parser.read(conf_file_name, encoding="utf-8") - params_dict = {} - for params in parser.items("DEFAULT"): - params_dict[params[0]] = params[1] - return params_dict - click.echo("Jira config file is required.") - raise click.Abort() - - -class JiraConnector: - def __init__(self, cfg_file): - self.cfg_file = cfg_file - config_dict = get_jira_connection_params(conf_file_name=self.cfg_file) - self.token = config_dict["token"] - self.server = config_dict["url"] - if not (self.token and self.server): - raise ValueError("Jira config file must contain token and server information.") - - self.resolved_statuses = config_dict.get('resolved_statuses') - self.project_ids = config_dict.get('project_ids') - self.jira = None - self.authenticate_to_jira_server() - - def authenticate_to_jira_server(self): - try: - self.jira = JIRA(token_auth=self.token, options={"server": self.server}) - LOGGER.info("Connected to Jira") - except JIRAError as e: - LOGGER.error("Failed to connect to Jira: %s", e) - raise - - def get_issue_metadata(self, jira_id, max_retry=3): - retries = 0 - while True: - try: - return self.jira.issue(id=jira_id, fields="status, issuetype, fixVersions").fields - except JIRAError as e: - # Check for inactivity error (adjust based on your library) - if "401 Unauthorized" in str(e) or "Session timed out" in str(e): - retries += 1 - LOGGER.warning("Failed to get issue due to inactivity, retrying (%d/%d)", retries, max_retry) - if retries < max_retry: - self.authenticate_to_jira_server() # Attempt reconnection - else: - raise # Re-raise the error after exceeding retries - else: - raise - - -def get_all_jiras_from_file(file_content): - """ - Try to find all jira_utils tickets in the file. - Looking for the following patterns: - - jira_id=CNV-12345 # call in is_jira_open - - https://issues.redhat.com/browse/CNV-12345 # when jira_utils is in a link in comments - - pytest.mark.jira_utils(CNV-12345) # when jira_utils is in a marker - - Args: - file_content (str): The content of the file. - - Returns: - list: A list of jira_utils tickets. - """ - issue_pattern = r"([A-Z]+-[0-9]+)" - _pytest_jira_marker_bugs = re.findall( - rf"pytest.mark.jira.*?{issue_pattern}.*", file_content, re.DOTALL - ) - _is_jira_open = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content) - _jira_url_jiras = re.findall( - rf"https://issues.redhat.com/browse/{issue_pattern}(?! )", - file_content, - ) - return list(set(_pytest_jira_marker_bugs + _is_jira_open + _jira_url_jiras)) - - -def get_jiras_from_python_files(): - jira_found = {} - for filename in all_python_files(): - with open(filename) as fd: - if unique_jiras := get_all_jiras_from_file(file_content=fd.read()): - jira_found[filename] = unique_jiras - LOGGER.warning(f"File: {filename}, {unique_jiras}") - return jira_found - - -def get_closed_jira_ids(jira_connector, jira_ids_dict): - jira_errors = {} - for file_name in jira_ids_dict: - for jira_id in jira_ids_dict[file_name]: - try: - current_jira_status = jira_connector.get_issue_metadata(jira_id=jira_id).status.name.lower() - if current_jira_status in jira_connector.resolved_statuses: - jira_errors.setdefault(file_name, []).append( - f"{jira_id} [{current_jira_status}]" - ) - except JIRAError as exp: - jira_errors.setdefault(file_name, []).append( - f"{jira_id} [{exp.text}]" - ) - continue - return jira_errors - - -def get_jira_version_mismatch(jira_connector, jira_id_dict, jira_expected_versions=None): - jira_mismatch = {} - for file_name in jira_id_dict: - unique_jira_ids = jira_id_dict[file_name] - if jira_connector.project_ids: - unique_jira_ids = [jira_id for jira_id in unique_jira_ids - if jira_id.startswith(tuple(jira_connector.project_ids))] - for jira_id in unique_jira_ids: - jira_issue = jira_connector.get_issue_metadata(jira_id=jira_id) - fix_version = re.search(r"([\d.]+)", jira_issue.fixVersions[0].name) if jira_issue.fixVersions else None - jira_target_release_version = fix_version.group(1) if fix_version else "vfuture" - LOGGER.info(f"issue: {jira_id}, version: {jira_target_release_version}, {jira_expected_versions}") - if not jira_target_release_version.startswith(tuple(jira_expected_versions)): - jira_mismatch.setdefault(file_name, []).append( - f"{jira_id} [{jira_target_release_version}]" - ) - - return jira_mismatch diff --git a/apps/utils.py b/apps/utils.py index b8fe356..852598b 100644 --- a/apps/utils.py +++ b/apps/utils.py @@ -80,7 +80,6 @@ def all_python_files() -> Iterable[str]: for root, _, files in os.walk(os.path.abspath(os.curdir)): if [_dir for _dir in exclude_dirs if _dir in root]: continue - LOGGER.info(root) for filename in files: if filename.endswith(".py"): yield os.path.join(root, filename) diff --git a/example.jira.cfg b/example.jira.cfg deleted file mode 100644 index 569fee4..0000000 --- a/example.jira.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] -url = https://www.atlassian.com/ -token = mytoken -resolved_statuses = (verified, release pending, closed) diff --git a/example.jira.yaml b/example.jira.yaml new file mode 100644 index 0000000..60dfdb7 --- /dev/null +++ b/example.jira.yaml @@ -0,0 +1,12 @@ +pyutils-jira: + url: + token: mytoken + resolved_statuses: + - verified + - release pending + - closed + skip_project_ids: + - ABC + - DEF + jira_target_versions: + - 1.0.0 diff --git a/poetry.lock b/poetry.lock index 65263fe..d3e6b7b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "appnope" @@ -563,31 +563,6 @@ docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alab qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] -[[package]] -name = "markdown-it-py" -version = "3.0.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.8" -files = [ - {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, - {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -code-style = ["pre-commit (>=3.0,<4.0)"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins"] -profiling = ["gprof2dot"] -rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - - [[package]] name = "jira" version = "3.6.0" @@ -615,6 +590,30 @@ docs = ["furo", "sphinx (>=5.0.0)", "sphinx-copybutton"] opt = ["PyJWT", "filemagic (>=1.6)", "requests-jwt", "requests-kerberos"] test = ["MarkupSafe (>=0.23)", "PyYAML (>=5.1)", "docutils (>=0.12)", "flaky", "oauthlib", "parameterized (>=0.8.1)", "pytest (>=6.0.0)", "pytest-cache", "pytest-cov", "pytest-instafail", "pytest-sugar", "pytest-timeout (>=1.3.1)", "pytest-xdist (>=2.2)", "requests-mock", "requires.io", "tenacity", "wheel (>=0.24.0)", "yanc (>=0.3.3)"] +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "matplotlib-inline" version = "0.1.6" @@ -654,6 +653,22 @@ files = [ [package.extras] nicer-shell = ["ipython"] +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + [[package]] name = "packaging" version = "23.2" @@ -1129,23 +1144,21 @@ socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] -name = "rich" -version = "13.7.1" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +name = "requests-oauthlib" +version = "2.0.0" +description = "OAuthlib authentication support for Requests." optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.4" files = [ - {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, - {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, + {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"}, + {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"}, ] [package.dependencies] -markdown-it-py = ">=2.2.0" -pygments = ">=2.13.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} +oauthlib = ">=3.0.0" +requests = ">=2.0.0" [package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] rsa = ["oauthlib[signedtoken] (>=3.0.0)"] [[package]] @@ -1162,6 +1175,38 @@ files = [ [package.dependencies] requests = ">=2.0.1,<3.0.0" +[[package]] +name = "retry2" +version = "0.9.5" +description = "Easy to use retry decorator." +optional = false +python-versions = ">=2.6" +files = [ + {file = "retry2-0.9.5-py2.py3-none-any.whl", hash = "sha256:f7fee13b1e15d0611c462910a6aa72a8919823988dd0412152bc3719c89a4e55"}, +] + +[package.dependencies] +decorator = ">=3.4.2" + +[[package]] +name = "rich" +version = "13.7.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, + {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + [[package]] name = "six" version = "1.16.0" diff --git a/pyproject.toml b/pyproject.toml index a0fbc71..3d6ca77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,12 +47,9 @@ packages = [{ include = "apps" }] [tool.poetry.scripts] pyutils-unusedcode = "apps.unused_code.unused_code:get_unused_functions" pyutils-polarion-verify-tc-requirements = "apps.polarion.polarion_verify_tc_requirements:has_verify" -<<<<<<< Updated upstream pyutils-polarion-set-automated = "apps.polarion.polarion_set_automated:polarion_approve_automate" -======= -pyutils-jira = "apps.jira_utils.get_jira_information:get_closed_jiras" +pyutils-jira = "apps.jira_utils.get_jira_information:get_jira_mismatch" ->>>>>>> Stashed changes [tool.poetry.dependencies] python = "^3.8" @@ -62,6 +59,7 @@ pyhelper-utils = "^0.0.41" pytest-mock = "^3.14.0" pyyaml = "^6.0.1" jira = "^3.6.0" +retry2 = "^0.9.5" [tool.poetry.group.dev.dependencies] ipdb = "^0.13.13" diff --git a/tests/jira_utils/test_jira.py b/tests/jira_utils/test_jira.py index 553f7c5..e92a42d 100644 --- a/tests/jira_utils/test_jira.py +++ b/tests/jira_utils/test_jira.py @@ -1,34 +1,21 @@ -from apps.jira_utils.get_jira_information import get_closed_jiras -from tests.utils import get_cli_runner -from simple_logger.logger import get_logger -from unittest import mock - -LOGGER = get_logger(name=__name__) +import shlex +import subprocess +from pyhelper_utils.shell import run_command -def test_jira_missing_target_version(): - result = get_cli_runner().invoke(get_closed_jiras, "--jira-cfg-file example.jira.cfg") - LOGGER.info(f"Result output: {result.output}, exit code: {result.exit_code}, exceptions: {result.exception}") - assert result.exit_code != 0 - assert "Missing option '--jira-target-versions'" in result.output - +from simple_logger.logger import get_logger -def test_jira_missing_cfg_file(): - result = get_cli_runner().invoke(get_closed_jiras, '--jira-cfg-file jira.cfg --jira-target-versions "ABC"') - LOGGER.info(f"Result output: {result.output}, exit code: {result.exit_code}, exceptions: {result.exception}") - assert result.exit_code == 1 - assert "Jira config file is required." in result.output +LOGGER = get_logger(name=__name__) +BASE_COMMAND = "poetry run python apps/jira_utils/get_jira_information.py --verbose" -def test_jira_with_closed_bugs(): - with mock.patch("apps.jira_utils.get_jira_information.JiraConnector") as jira_connector: - a_instance = jira_connector.return_value - a_instance.get_issue_metadata.return_value = {'status':{'name':'closed'}} - with mock.patch("apps.jira_utils.get_jira_information.get_jiras_from_python_files") as jiras: - jiras.return_value = {"file1": ['ABC-1234']} - with mock.patch("apps.jira_utils.get_jira_information.JiraConnector.get_closed_jira_ids") as closed_ids: - closed_ids.return_value = "{'file1': ['ABC-1234' ['closed']]}" - result = get_cli_runner().invoke(get_closed_jiras, '--jira-target-versions "ABC"') - assert result.exit_code == 1 - #assert f"Following jiras are not open or could not be accessed:" in result.output - LOGGER.info(f"Result output: {result.output}, exit code: {result.exit_code}, exceptions: {result.exception}") +def test_jira_missing_config_file(): + rc, _, err = run_command( + command=shlex.split(f"{BASE_COMMAND} --jira-cfg-file invalid-jira.cfg"), + verify_stderr=False, + check=False, + capture_output=False, + stderr=subprocess.PIPE, + ) + assert "Jira config file must contain valid url and token" in err + assert not rc From a583779247c58c36b91d1fda6f41dae42f8b9347 Mon Sep 17 00:00:00 2001 From: Debarati Basu-Nag Date: Fri, 23 Aug 2024 11:02:40 -0400 Subject: [PATCH 04/17] Update apps/jira_utils/get_jira_information.py Co-authored-by: Meni Yakove <441263+myakove@users.noreply.github.com> --- apps/jira_utils/get_jira_information.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py index 60186ba..090d1b4 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/get_jira_information.py @@ -41,7 +41,7 @@ def get_jira_mismatch(jira_cfg_file: Any, jira_target_versions: Any, verbose: bo raise JiraInvalidConfigFileError("Jira config file must contain valid url and token.") jira_connector = JiraConnector(token=config_dict["token"], url=config_dict["url"]) jira_error: Dict[str, Any] = {"status_mismatch": {}, "version_mismatch": {}, "connection_error": {}} - resolved_status = config_dict.get("resolved_statuses") or DEFAULT_RESOLVED_STATUS + resolved_status = config_dict.get("resolved_statuses", DEFAULT_RESOLVED_STATUS) jira_target_versions = jira_target_versions or config_dict.get("jira_target_versions", []) skip_project_ids = config_dict.get("skip_project_ids", []) for file_name in (jira_id_dict := get_jiras_from_python_files()): From 059182da188290033e77043557a3ebb47da2e636 Mon Sep 17 00:00:00 2001 From: Debarati Basu-Nag Date: Fri, 23 Aug 2024 11:02:52 -0400 Subject: [PATCH 05/17] Update apps/jira_utils/get_jira_information.py Co-authored-by: Meni Yakove <441263+myakove@users.noreply.github.com> --- apps/jira_utils/get_jira_information.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py index 090d1b4..9c1113a 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/get_jira_information.py @@ -40,7 +40,7 @@ def get_jira_mismatch(jira_cfg_file: Any, jira_target_versions: Any, verbose: bo if not (config_dict.get("url") and config_dict.get("token")): raise JiraInvalidConfigFileError("Jira config file must contain valid url and token.") jira_connector = JiraConnector(token=config_dict["token"], url=config_dict["url"]) - jira_error: Dict[str, Any] = {"status_mismatch": {}, "version_mismatch": {}, "connection_error": {}} + jira_error: Dict[str, Dict[str, Any]] = {"status_mismatch": {}, "version_mismatch": {}, "connection_error": {}} resolved_status = config_dict.get("resolved_statuses", DEFAULT_RESOLVED_STATUS) jira_target_versions = jira_target_versions or config_dict.get("jira_target_versions", []) skip_project_ids = config_dict.get("skip_project_ids", []) From e4b50c701bf76b7d49fa189663787c6e63b5e201 Mon Sep 17 00:00:00 2001 From: dbasunag Date: Fri, 23 Aug 2024 16:13:23 -0400 Subject: [PATCH 06/17] address review comments --- apps/jira_utils/README.md | 1 + apps/jira_utils/get_jira_information.py | 23 ++++++++++++----- apps/jira_utils/jira_utils.py | 34 ++++++++++++++++--------- config.example.yaml | 8 ++++++ poetry.lock | 28 ++++++++++---------- pyproject.toml | 2 +- 6 files changed, 64 insertions(+), 32 deletions(-) diff --git a/apps/jira_utils/README.md b/apps/jira_utils/README.md index 8cc550d..874d8c4 100644 --- a/apps/jira_utils/README.md +++ b/apps/jira_utils/README.md @@ -34,6 +34,7 @@ pyutils-jira: jira_target_versions: - 1.0.0 - 2.0.1 + issue_pattern: "([A-Z]+-[0-9]+)" ``` This would skip version checks on any jira ids associated with project ABC and DEF This would also check if the current repository branch is pointing to any jira card that is not targeted for 1.0.0 or 2.0.1 version diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py index 9c1113a..b9cba11 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/get_jira_information.py @@ -30,21 +30,32 @@ type=ListParamType(), required=False, ) +@click.option( + "--jira-issue-pattern", + help="Provide the regex for Jira ids, default is ([A-Z]+-[0-9]+)", + type=click.STRING, + default="([A-Z]+-[0-9]+)", +) @click.option("--verbose", default=False, is_flag=True) -def get_jira_mismatch(jira_cfg_file: Any, jira_target_versions: Any, verbose: bool) -> Any: +def get_jira_mismatch( + jira_cfg_file: str, jira_target_versions: list[str], jira_issue_pattern: str, verbose: bool +) -> None: if verbose: LOGGER.setLevel(logging.DEBUG) else: logging.disable(logging.CRITICAL) config_dict = get_util_config(util_name="pyutils-jira", config_file_path=jira_cfg_file) - if not (config_dict.get("url") and config_dict.get("token")): - raise JiraInvalidConfigFileError("Jira config file must contain valid url and token.") - jira_connector = JiraConnector(token=config_dict["token"], url=config_dict["url"]) + jira_url = config_dict.get("url") + jira_token = config_dict.get("token") + jira_issue_pattern = config_dict.get("issue_pattern", jira_issue_pattern) + if not (jira_url and jira_token and jira_issue_pattern): + raise JiraInvalidConfigFileError("Jira config file must contain valid url, token or issue pattern.") + jira_connector = JiraConnector(token=jira_token, url=jira_url) jira_error: Dict[str, Dict[str, Any]] = {"status_mismatch": {}, "version_mismatch": {}, "connection_error": {}} - resolved_status = config_dict.get("resolved_statuses", DEFAULT_RESOLVED_STATUS) + resolved_status = config_dict.get("resolved_statuses", DEFAULT_RESOLVED_STATUS) jira_target_versions = jira_target_versions or config_dict.get("jira_target_versions", []) skip_project_ids = config_dict.get("skip_project_ids", []) - for file_name in (jira_id_dict := get_jiras_from_python_files()): + for file_name in (jira_id_dict := get_jiras_from_python_files(issue_pattern=jira_issue_pattern)): for jira_id in jira_id_dict[file_name]: try: # check closed status: diff --git a/apps/jira_utils/jira_utils.py b/apps/jira_utils/jira_utils.py index 23fe73c..6f3a25b 100644 --- a/apps/jira_utils/jira_utils.py +++ b/apps/jira_utils/jira_utils.py @@ -1,7 +1,7 @@ import re from simple_logger.logger import get_logger from jira import JIRA, JIRAError, Issue -from retry import retry +from tenacity import retry, retry_if_exception_type, wait_fixed, stop_after_attempt from apps.utils import all_python_files from typing import List, Dict, Any @@ -14,12 +14,13 @@ def __init__(self, token: str, url: str) -> None: self.url = url self.jira = JIRA(token_auth=self.token, options={"server": self.url}) - @retry(JIRAError, tries=3, delay=2) + @retry(retry=retry_if_exception_type(JIRAError), stop=stop_after_attempt(3), wait=wait_fixed(2)) def get_issue(self, jira_id: str) -> Issue: + LOGGER.info(f"Retry staistics: {self.get_issue.statistics}") return self.jira.issue(id=jira_id, fields="status, issuetype, fixVersions") -def get_jira_ids_from_file_content(file_content: str) -> List[str]: +def get_jira_ids_from_file_content(file_content: str, issue_pattern: str) -> List[str]: """ Try to find all jira_utils tickets in a given file content. Looking for the following patterns: @@ -29,36 +30,45 @@ def get_jira_ids_from_file_content(file_content: str) -> List[str]: Args: file_content (str): The content of a given file. + issue_pattern (str): regex pattern for jira ids Returns: list: A list of jira tickets. """ - issue_pattern = r"([A-Z]+-[0-9]+)" _pytest_jira_marker_bugs = re.findall(rf"pytest.mark.jira.*?{issue_pattern}.*", file_content, re.DOTALL) - _is_jira_open = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content) - _jira_url_jiras = [] + _jira_id_arguments = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content) _jira_url_jiras = re.findall( rf"https://issues.redhat.com/browse/{issue_pattern}", file_content, ) - return list(set(_pytest_jira_marker_bugs + _is_jira_open + _jira_url_jiras)) + return list(set(_pytest_jira_marker_bugs + _jira_id_arguments + _jira_url_jiras)) -def get_jiras_from_python_files() -> Dict[str, Any]: +def get_jiras_from_python_files(issue_pattern: str) -> Dict[str, Any]: """ Get all python files from the current directory and get list of jira ids from each of them + Args: + issue_pattern (str): regex pattern for jira ids + + Returns: + Dict: A dict of filenames and associated jira tickets. + Note: any line containing would be not be checked for presence of a jira id """ - jira_found = {} + jira_found: Dict[str, list[str]] = {} for filename in all_python_files(): with open(filename) as fd: file_content = [] for line in fd.readlines(): # if appears in a line, exclude that line from jira check - if "" not in line: + if "" in line: + continue + else: file_content.append(line) - if unique_jiras := get_jira_ids_from_file_content(file_content="\n".join(file_content)): + if unique_jiras := get_jira_ids_from_file_content( + file_content="\n".join(file_content), issue_pattern=issue_pattern + ): jira_found[filename] = unique_jiras - LOGGER.info(f"File: {filename}, {unique_jiras}") + LOGGER.info(f"Following jiras are found: {jira_found}") return jira_found diff --git a/config.example.yaml b/config.example.yaml index 8decab9..830dc1e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -9,3 +9,11 @@ pyutils-polarion-verify-tc-requirements: pyutils-polarion-set-automated: project_id: "ABCDEF" branch: "origin/main" +pyutils-jira: + url: + token: + resolved_statuses: + - verified + - release pending + - closed + issue_pattern: "([A-Z]+-[0-9]+)" diff --git a/poetry.lock b/poetry.lock index d3e6b7b..398cccb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1175,19 +1175,6 @@ files = [ [package.dependencies] requests = ">=2.0.1,<3.0.0" -[[package]] -name = "retry2" -version = "0.9.5" -description = "Easy to use retry decorator." -optional = false -python-versions = ">=2.6" -files = [ - {file = "retry2-0.9.5-py2.py3-none-any.whl", hash = "sha256:f7fee13b1e15d0611c462910a6aa72a8919823988dd0412152bc3719c89a4e55"}, -] - -[package.dependencies] -decorator = ">=3.4.2" - [[package]] name = "rich" version = "13.7.1" @@ -1248,6 +1235,21 @@ files = [ {file = "suds-1.1.2.tar.gz", hash = "sha256:1d5cfa74117193b244a4233f246c483d9f41198b448c5f14a8bad11c4f649f2b"}, ] +[[package]] +name = "tenacity" +version = "9.0.0" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539"}, + {file = "tenacity-9.0.0.tar.gz", hash = "sha256:807f37ca97d62aa361264d497b0e31e92b8027044942bfa756160d908320d73b"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "tomli" version = "2.0.1" diff --git a/pyproject.toml b/pyproject.toml index 3d6ca77..3e0322f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ pyhelper-utils = "^0.0.41" pytest-mock = "^3.14.0" pyyaml = "^6.0.1" jira = "^3.6.0" -retry2 = "^0.9.5" +tenacity = "^9.0.0" [tool.poetry.group.dev.dependencies] ipdb = "^0.13.13" From f219a6db159f5a7a4c6c924474f2ec256d646b08 Mon Sep 17 00:00:00 2001 From: dbasunag Date: Fri, 23 Aug 2024 16:20:07 -0400 Subject: [PATCH 07/17] fix typo --- apps/jira_utils/README.md | 2 +- apps/jira_utils/jira_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/jira_utils/README.md b/apps/jira_utils/README.md index 874d8c4..61aca87 100644 --- a/apps/jira_utils/README.md +++ b/apps/jira_utils/README.md @@ -45,7 +45,7 @@ To run from CLI with `--jira-target-versions` pyutils-unusedcode --jira-target-versions '1.0.0,2.0.1' ``` #### Note: -To mark to skip a jira from these checks, one can add `` as a comment to the same line containing the specific jira +To mark to skip a jira from these checks, one can add `` as a comment to the same line containing the specific jira Example: ```bash diff --git a/apps/jira_utils/jira_utils.py b/apps/jira_utils/jira_utils.py index 6f3a25b..c0edce3 100644 --- a/apps/jira_utils/jira_utils.py +++ b/apps/jira_utils/jira_utils.py @@ -61,8 +61,8 @@ def get_jiras_from_python_files(issue_pattern: str) -> Dict[str, Any]: with open(filename) as fd: file_content = [] for line in fd.readlines(): - # if appears in a line, exclude that line from jira check - if "" in line: + # if appears in a line, exclude that line from jira check + if "" in line: continue else: file_content.append(line) From 688c9f4eabf2ae2a5d2e745e51f0393188f5d30e Mon Sep 17 00:00:00 2001 From: Debarati Basu-Nag Date: Mon, 26 Aug 2024 15:19:37 -0400 Subject: [PATCH 08/17] Update apps/jira_utils/get_jira_information.py Co-authored-by: Ruth Netser --- apps/jira_utils/get_jira_information.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py index b9cba11..574ff6a 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/get_jira_information.py @@ -58,7 +58,7 @@ def get_jira_mismatch( for file_name in (jira_id_dict := get_jiras_from_python_files(issue_pattern=jira_issue_pattern)): for jira_id in jira_id_dict[file_name]: try: - # check closed status: + # check resolved status: jira_issue_metadata = jira_connector.get_issue(jira_id=jira_id).fields current_jira_status = jira_issue_metadata.status.name.lower() if current_jira_status in resolved_status: From ca1f26b465116fdc645bcb7970e3b3f6da7ce49e Mon Sep 17 00:00:00 2001 From: Debarati Basu-Nag Date: Tue, 27 Aug 2024 19:08:25 -0400 Subject: [PATCH 09/17] Update apps/jira_utils/README.md Co-authored-by: Ruth Netser --- apps/jira_utils/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/jira_utils/README.md b/apps/jira_utils/README.md index 61aca87..fc32535 100644 --- a/apps/jira_utils/README.md +++ b/apps/jira_utils/README.md @@ -42,7 +42,7 @@ This would also check if the current repository branch is pointing to any jira c To run from CLI with `--jira-target-versions` ```bash -pyutils-unusedcode --jira-target-versions '1.0.0,2.0.1' +pyutils-jira --jira-target-versions '1.0.0,2.0.1' ``` #### Note: To mark to skip a jira from these checks, one can add `` as a comment to the same line containing the specific jira From 9a27109d7b7d683e7f5efa6f556bbb73ca316fcc Mon Sep 17 00:00:00 2001 From: dbasunag Date: Tue, 27 Aug 2024 20:23:22 -0400 Subject: [PATCH 10/17] updates based on review comments --- apps/jira_utils/README.md | 16 ++++++-- apps/jira_utils/exceptions.py | 6 --- apps/jira_utils/get_jira_information.py | 54 ++++++++++++++++--------- apps/jira_utils/jira_utils.py | 48 +++++++++++----------- config.example.yaml | 6 +++ example.jira.yaml | 12 ------ 6 files changed, 78 insertions(+), 64 deletions(-) delete mode 100644 apps/jira_utils/exceptions.py delete mode 100644 example.jira.yaml diff --git a/apps/jira_utils/README.md b/apps/jira_utils/README.md index fc32535..ddb0367 100644 --- a/apps/jira_utils/README.md +++ b/apps/jira_utils/README.md @@ -3,7 +3,7 @@ Helper to identify wrong jira card references in a repository files. e.g. closed It looks for following patterns in files of a pytest repository: ```bash - jira_id=ABC-12345 # When jira id is present in a function call - - https://issues.redhat.com/browse/ABC-12345 # when jira is in a link in comments + - /browse/ABC-12345 # when jira is in a link in comments - pytest.mark.jira_utils(ABC-12345) # when jira is in a marker ``` @@ -15,8 +15,8 @@ pyutils-jira --help ``` ## Config file -A config file with the jira connection parameters like url, token, resolved_statuses, skip_project_ids, jira_target_versions should be added to -`~/.config/python-utility-scripts/jira_utils/config.cfg` +A config file with the jira connection parameters like url, token, resolved_statuses, skip_project_ids, jira_target_versions should be passed to command line option `--jira-cfg-file` +By default config file location `~/.config/python-utility-scripts/jira_utils/config.cfg` is used, but the location can be overwritten by using the `--jira-cfg-file` option. ### Example: @@ -35,6 +35,7 @@ pyutils-jira: - 1.0.0 - 2.0.1 issue_pattern: "([A-Z]+-[0-9]+)" + version_string_not_targeted_jiras: "vfuture" ``` This would skip version checks on any jira ids associated with project ABC and DEF This would also check if the current repository branch is pointing to any jira card that is not targeted for 1.0.0 or 2.0.1 version @@ -49,5 +50,12 @@ To mark to skip a jira from these checks, one can add `` Example: ```bash - #https://issues.redhat.com/browse/OSD-5716 + #https://issues.redhat.com/browse/OSD-5716 ``` +If resolved_statuses is not provided in config file, by default the following status would be considered as resolved statuses: + +```bash +["verified", "release pending", "closed", "resolved"] +``` + +Default value for `version_string_not_targeted_jiras` is "vfuture" diff --git a/apps/jira_utils/exceptions.py b/apps/jira_utils/exceptions.py deleted file mode 100644 index 7521d32..0000000 --- a/apps/jira_utils/exceptions.py +++ /dev/null @@ -1,6 +0,0 @@ -class JiraInvalidConfigFileError(Exception): - pass - - -class JiraValidationError(Exception): - pass diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py index 574ff6a..3b96368 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/get_jira_information.py @@ -3,14 +3,19 @@ import re import click -from jira import JIRAError +from jira import JIRAError, JIRA from simple_logger.logger import get_logger -from apps.jira_utils.exceptions import JiraValidationError, JiraInvalidConfigFileError -from apps.jira_utils.jira_utils import get_jiras_from_python_files, JiraConnector +from apps.jira_utils.jira_utils import ( + get_jiras_from_python_files, + JiraInvalidConfigFileError, + JiraValidationError, + get_issue, +) + from apps.utils import ListParamType, get_util_config -from typing import Any, Dict +from typing import Any, Dict, List DEFAULT_RESOLVED_STATUS = ["verified", "release pending", "closed", "resolved"] @@ -21,7 +26,7 @@ @click.option( "--jira-cfg-file", help="Provide absolute path to the jira_utils config file. ", - type=click.Path(), + type=click.Path(exists=True), default=os.path.expanduser("~/.config/python-utility-scripts/jira_utils/config.cfg"), ) @click.option( @@ -32,13 +37,25 @@ ) @click.option( "--jira-issue-pattern", - help="Provide the regex for Jira ids, default is ([A-Z]+-[0-9]+)", + help="Provide the regex for Jira ids", type=click.STRING, + show_default=True, default="([A-Z]+-[0-9]+)", ) +@click.option( + "--version-string-not-targeted-jiras", + help="Provide possible version strings for not yet targeted jiras", + type=click.STRING, + show_default=True, + default="vfuture", +) @click.option("--verbose", default=False, is_flag=True) def get_jira_mismatch( - jira_cfg_file: str, jira_target_versions: list[str], jira_issue_pattern: str, verbose: bool + jira_cfg_file: str, + jira_target_versions: List[str], + jira_issue_pattern: str, + version_string_not_targeted_jiras: str, + verbose: bool, ) -> None: if verbose: LOGGER.setLevel(logging.DEBUG) @@ -48,21 +65,22 @@ def get_jira_mismatch( jira_url = config_dict.get("url") jira_token = config_dict.get("token") jira_issue_pattern = config_dict.get("issue_pattern", jira_issue_pattern) - if not (jira_url and jira_token and jira_issue_pattern): - raise JiraInvalidConfigFileError("Jira config file must contain valid url, token or issue pattern.") - jira_connector = JiraConnector(token=jira_token, url=jira_url) - jira_error: Dict[str, Dict[str, Any]] = {"status_mismatch": {}, "version_mismatch": {}, "connection_error": {}} + if not (jira_url and jira_token): + raise JiraInvalidConfigFileError("Jira config file must contain valid url or token.") + jira_connection = JIRA(token_auth=jira_token, options={"server": jira_url}) + jira_error: Dict[str, Dict[str, Any]] = {} resolved_status = config_dict.get("resolved_statuses", DEFAULT_RESOLVED_STATUS) jira_target_versions = jira_target_versions or config_dict.get("jira_target_versions", []) skip_project_ids = config_dict.get("skip_project_ids", []) - for file_name in (jira_id_dict := get_jiras_from_python_files(issue_pattern=jira_issue_pattern)): + not_targeted_version_str = config_dict.get("version_string_not_targeted_jiras", version_string_not_targeted_jiras) + for file_name in (jira_id_dict := get_jiras_from_python_files(issue_pattern=jira_issue_pattern, jira_url=jira_url)): for jira_id in jira_id_dict[file_name]: try: # check resolved status: - jira_issue_metadata = jira_connector.get_issue(jira_id=jira_id).fields + jira_issue_metadata = get_issue(jira=jira_connection, jira_id=jira_id).fields current_jira_status = jira_issue_metadata.status.name.lower() if current_jira_status in resolved_status: - jira_error["status_mismatch"].setdefault(file_name, []).append( + jira_error.setdefault("status_mismatch", {}).setdefault(file_name, []).append( f"{jira_id}: current status: {current_jira_status}" ) # validate correct target version if provided: @@ -74,17 +92,17 @@ def get_jira_mismatch( if (jira_issue_metadata.fixVersions) else None ) - current_target_version = fix_version.group(1) if fix_version else "vfuture" + current_target_version = fix_version.group(1) if fix_version else not_targeted_version_str if not any([current_target_version == version for version in jira_target_versions]): - jira_error["version_mismatch"].setdefault(file_name, []).append( + jira_error.setdefault("version_mismatch", {}).setdefault(file_name, []).append( f"{jira_id}: target version: {current_target_version}]" ) except JIRAError as exp: - jira_error["connection_error"].setdefault(file_name, []).append( + jira_error.setdefault("connection_error", {}).setdefault(file_name, []).append( f"{jira_id}: status code: {exp.status_code}, details: {exp.text}]." ) - jira_error = {key: value for key, value in jira_error.items() if value} + # https://issues.redhat.com/browse/RHEL-40899 if jira_error.values(): error = "Following Jira ids failed jira check:\n" if jira_error.get("status_mismatch"): diff --git a/apps/jira_utils/jira_utils.py b/apps/jira_utils/jira_utils.py index c0edce3..b15873b 100644 --- a/apps/jira_utils/jira_utils.py +++ b/apps/jira_utils/jira_utils.py @@ -8,24 +8,18 @@ LOGGER = get_logger(name=__name__) -class JiraConnector: - def __init__(self, token: str, url: str) -> None: - self.token = token - self.url = url - self.jira = JIRA(token_auth=self.token, options={"server": self.url}) +@retry(retry=retry_if_exception_type(JIRAError), stop=stop_after_attempt(3), wait=wait_fixed(2)) +def get_issue(jira: JIRA, jira_id: str) -> Issue: + LOGGER.info(f"Retry staistics for {jira_id}: {get_issue.statistics}") + return jira.issue(id=jira_id, fields="status, issuetype, fixVersions") - @retry(retry=retry_if_exception_type(JIRAError), stop=stop_after_attempt(3), wait=wait_fixed(2)) - def get_issue(self, jira_id: str) -> Issue: - LOGGER.info(f"Retry staistics: {self.get_issue.statistics}") - return self.jira.issue(id=jira_id, fields="status, issuetype, fixVersions") - -def get_jira_ids_from_file_content(file_content: str, issue_pattern: str) -> List[str]: +def get_jira_ids_from_file_content(file_content: str, issue_pattern: str, jira_url: str) -> List[str]: """ - Try to find all jira_utils tickets in a given file content. + Try to find all Jira tickets in a given file content. Looking for the following patterns: - jira_id=ABC-12345 # When jira id is present in a function call - - https://issues.redhat.com/browse/ABC-12345 # when jira is in a link in comments + - /browse/ABC-12345 # when jira is in a link in comments - pytest.mark.jira_utils(ABC-12345) # when jira is in a marker Args: @@ -38,18 +32,19 @@ def get_jira_ids_from_file_content(file_content: str, issue_pattern: str) -> Lis _pytest_jira_marker_bugs = re.findall(rf"pytest.mark.jira.*?{issue_pattern}.*", file_content, re.DOTALL) _jira_id_arguments = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content) _jira_url_jiras = re.findall( - rf"https://issues.redhat.com/browse/{issue_pattern}", + rf"{jira_url}/browse/{issue_pattern}", file_content, ) return list(set(_pytest_jira_marker_bugs + _jira_id_arguments + _jira_url_jiras)) -def get_jiras_from_python_files(issue_pattern: str) -> Dict[str, Any]: +def get_jiras_from_python_files(issue_pattern: str, jira_url: str) -> Dict[str, Any]: """ Get all python files from the current directory and get list of jira ids from each of them Args: issue_pattern (str): regex pattern for jira ids + jira_url (str): jira url that could be used to look for possible presence of jira references in a file Returns: Dict: A dict of filenames and associated jira tickets. @@ -59,16 +54,21 @@ def get_jiras_from_python_files(issue_pattern: str) -> Dict[str, Any]: jira_found: Dict[str, list[str]] = {} for filename in all_python_files(): with open(filename) as fd: - file_content = [] - for line in fd.readlines(): - # if appears in a line, exclude that line from jira check - if "" in line: - continue - else: - file_content.append(line) + # if appears in a line, exclude that line from jira check if unique_jiras := get_jira_ids_from_file_content( - file_content="\n".join(file_content), issue_pattern=issue_pattern + file_content="\n".join([line for line in fd.readlines() if "" not in line]), + issue_pattern=issue_pattern, + jira_url=jira_url, ): jira_found[filename] = unique_jiras - LOGGER.info(f"Following jiras are found: {jira_found}") + if jira_found: + LOGGER.info(f"Following jiras are found: {jira_found}") return jira_found + + +class JiraInvalidConfigFileError(Exception): + pass + + +class JiraValidationError(Exception): + pass diff --git a/config.example.yaml b/config.example.yaml index 830dc1e..f9972cf 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -17,3 +17,9 @@ pyutils-jira: - release pending - closed issue_pattern: "([A-Z]+-[0-9]+)" + skip_project_ids: + - ABC + - DEF + jira_target_versions: + - 1.0.0 + version_string_not_targeted_jiras: "not-targeted" diff --git a/example.jira.yaml b/example.jira.yaml deleted file mode 100644 index 60dfdb7..0000000 --- a/example.jira.yaml +++ /dev/null @@ -1,12 +0,0 @@ -pyutils-jira: - url: - token: mytoken - resolved_statuses: - - verified - - release pending - - closed - skip_project_ids: - - ABC - - DEF - jira_target_versions: - - 1.0.0 From 83cf608a51910719810908d66c33cb2fc5654b98 Mon Sep 17 00:00:00 2001 From: dbasunag Date: Tue, 17 Sep 2024 11:55:26 -0400 Subject: [PATCH 11/17] remove jira tests, it would be covered in a follow up PR --- apps/jira_utils/get_jira_information.py | 117 +++++++++++++----------- apps/jira_utils/jira_utils.py | 49 ++++++++-- poetry.lock | 2 +- pyproject.toml | 2 +- tests/jira_utils/__init__.py | 0 tests/jira_utils/test_jira.py | 21 ----- 6 files changed, 108 insertions(+), 83 deletions(-) delete mode 100644 tests/jira_utils/__init__.py delete mode 100644 tests/jira_utils/test_jira.py diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py index 3b96368..fca0641 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/get_jira_information.py @@ -1,9 +1,9 @@ import logging import os -import re +import concurrent.futures import click -from jira import JIRAError, JIRA +from jira import JIRA from simple_logger.logger import get_logger @@ -11,13 +11,12 @@ get_jiras_from_python_files, JiraInvalidConfigFileError, JiraValidationError, - get_issue, + get_jira_information, ) from apps.utils import ListParamType, get_util_config -from typing import Any, Dict, List +from typing import Dict, List -DEFAULT_RESOLVED_STATUS = ["verified", "release pending", "closed", "resolved"] LOGGER = get_logger(name=__name__) @@ -35,6 +34,13 @@ type=ListParamType(), required=False, ) +@click.option( + "--jira-skip-projects", + help="Provide comma separated list of Jira Project keys, against which version check should be skipped.", + type=ListParamType(), +) +@click.option("--jira-url", help="Provide the Jira server URL", type=click.STRING, default=os.getenv("JIRA_SERVER_URL")) +@click.option("--jira-token", help="Provide the Jira token.", type=click.STRING, default=os.getenv("JIRA_TOKEN")) @click.option( "--jira-issue-pattern", help="Provide the regex for Jira ids", @@ -42,6 +48,13 @@ show_default=True, default="([A-Z]+-[0-9]+)", ) +@click.option( + "--jira-resolved-statuses", + help="Comma separated list of Jira resolved statuses", + type=ListParamType(), + show_default=True, + default="verified, release pending, closed, resolved", +) @click.option( "--version-string-not-targeted-jiras", help="Provide possible version strings for not yet targeted jiras", @@ -53,6 +66,10 @@ def get_jira_mismatch( jira_cfg_file: str, jira_target_versions: List[str], + jira_url: str, + jira_token: str, + jira_skip_projects: List[str], + jira_resolved_statuses: List[str], jira_issue_pattern: str, version_string_not_targeted_jiras: str, verbose: bool, @@ -61,59 +78,51 @@ def get_jira_mismatch( LOGGER.setLevel(logging.DEBUG) else: logging.disable(logging.CRITICAL) + jira_mismatch: Dict[str, str] = {} + # Process all the arguments passed from command line or config file or environment variable config_dict = get_util_config(util_name="pyutils-jira", config_file_path=jira_cfg_file) - jira_url = config_dict.get("url") - jira_token = config_dict.get("token") - jira_issue_pattern = config_dict.get("issue_pattern", jira_issue_pattern) + jira_url = jira_url or config_dict.get("url", "") + jira_token = jira_token or config_dict.get("token", "") if not (jira_url and jira_token): raise JiraInvalidConfigFileError("Jira config file must contain valid url or token.") - jira_connection = JIRA(token_auth=jira_token, options={"server": jira_url}) - jira_error: Dict[str, Dict[str, Any]] = {} - resolved_status = config_dict.get("resolved_statuses", DEFAULT_RESOLVED_STATUS) - jira_target_versions = jira_target_versions or config_dict.get("jira_target_versions", []) - skip_project_ids = config_dict.get("skip_project_ids", []) + + jira_issue_pattern = jira_issue_pattern or config_dict.get("issue_pattern", "") + resolved_status = jira_resolved_statuses or config_dict.get("resolved_statuses", []) not_targeted_version_str = config_dict.get("version_string_not_targeted_jiras", version_string_not_targeted_jiras) - for file_name in (jira_id_dict := get_jiras_from_python_files(issue_pattern=jira_issue_pattern, jira_url=jira_url)): - for jira_id in jira_id_dict[file_name]: - try: - # check resolved status: - jira_issue_metadata = get_issue(jira=jira_connection, jira_id=jira_id).fields - current_jira_status = jira_issue_metadata.status.name.lower() - if current_jira_status in resolved_status: - jira_error.setdefault("status_mismatch", {}).setdefault(file_name, []).append( - f"{jira_id}: current status: {current_jira_status}" - ) - # validate correct target version if provided: - if jira_target_versions: - if skip_project_ids and jira_id.startswith(tuple(skip_project_ids)): - continue - fix_version = ( - re.search(r"([\d.]+)", jira_issue_metadata.fixVersions[0].name) - if (jira_issue_metadata.fixVersions) - else None - ) - current_target_version = fix_version.group(1) if fix_version else not_targeted_version_str - if not any([current_target_version == version for version in jira_target_versions]): - jira_error.setdefault("version_mismatch", {}).setdefault(file_name, []).append( - f"{jira_id}: target version: {current_target_version}]" - ) - - except JIRAError as exp: - jira_error.setdefault("connection_error", {}).setdefault(file_name, []).append( - f"{jira_id}: status code: {exp.status_code}, details: {exp.text}]." - ) - # https://issues.redhat.com/browse/RHEL-40899 - if jira_error.values(): - error = "Following Jira ids failed jira check:\n" - if jira_error.get("status_mismatch"): - error += f" Jira ids in resolved state: {jira_error['status_mismatch']}." - if jira_error.get("version_mismatch"): - error += ( - f" Jira expected versions: {jira_target_versions}, " - f"current versions: {jira_error['version_mismatch']}." - ) - if jira_error.get("connection_error"): - error += f" Jira ids with connection error: {jira_error['connection_error']}." + jira_target_versions = jira_target_versions or config_dict.get("jira_target_versions", []) + skip_project_ids = jira_skip_projects or config_dict.get("skip_project_ids", []) + + jira_obj = JIRA(token_auth=jira_token, options={"server": jira_url}) + jira_error: Dict[str, str] = {} + + if jira_id_dict := get_jiras_from_python_files(issue_pattern=jira_issue_pattern, jira_url=jira_url): + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + future_to_jiras = { + executor.submit( + get_jira_information, + jira_object=jira_obj, + jira_id=jira_id, + skip_project_ids=skip_project_ids, + resolved_status=resolved_status, + jira_target_versions=jira_target_versions, + target_version_str=not_targeted_version_str, + ): jira_id + for jira_id in set.union(*jira_id_dict.values()) + } + + for future in concurrent.futures.as_completed(future_to_jiras): + jira_id = future_to_jiras[future] + jira_error_string = future.result() + if jira_error_string: + jira_error[jira_id] = jira_error_string + + for file_name, jiras in jira_id_dict.items(): + for jira_id in jiras: + if jira_error.get(jira_id): + jira_mismatch[file_name] = jira_error[jira_id] + + if jira_mismatch: + error = f"Following Jira ids failed jira check: {jira_mismatch}\n" LOGGER.error(error) raise JiraValidationError(error) LOGGER.info("Successfully completed Jira validations") diff --git a/apps/jira_utils/jira_utils.py b/apps/jira_utils/jira_utils.py index b15873b..70e36db 100644 --- a/apps/jira_utils/jira_utils.py +++ b/apps/jira_utils/jira_utils.py @@ -3,7 +3,7 @@ from jira import JIRA, JIRAError, Issue from tenacity import retry, retry_if_exception_type, wait_fixed, stop_after_attempt from apps.utils import all_python_files -from typing import List, Dict, Any +from typing import Dict, Set, List LOGGER = get_logger(name=__name__) @@ -14,7 +14,7 @@ def get_issue(jira: JIRA, jira_id: str) -> Issue: return jira.issue(id=jira_id, fields="status, issuetype, fixVersions") -def get_jira_ids_from_file_content(file_content: str, issue_pattern: str, jira_url: str) -> List[str]: +def get_jira_ids_from_file_content(file_content: str, issue_pattern: str, jira_url: str) -> Set[str]: """ Try to find all Jira tickets in a given file content. Looking for the following patterns: @@ -27,7 +27,7 @@ def get_jira_ids_from_file_content(file_content: str, issue_pattern: str, jira_u issue_pattern (str): regex pattern for jira ids Returns: - list: A list of jira tickets. + set: A set of jira tickets. """ _pytest_jira_marker_bugs = re.findall(rf"pytest.mark.jira.*?{issue_pattern}.*", file_content, re.DOTALL) _jira_id_arguments = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content) @@ -35,10 +35,10 @@ def get_jira_ids_from_file_content(file_content: str, issue_pattern: str, jira_u rf"{jira_url}/browse/{issue_pattern}", file_content, ) - return list(set(_pytest_jira_marker_bugs + _jira_id_arguments + _jira_url_jiras)) + return set(_pytest_jira_marker_bugs + _jira_id_arguments + _jira_url_jiras) -def get_jiras_from_python_files(issue_pattern: str, jira_url: str) -> Dict[str, Any]: +def get_jiras_from_python_files(issue_pattern: str, jira_url: str) -> Dict[str, Set[str]]: """ Get all python files from the current directory and get list of jira ids from each of them @@ -51,8 +51,9 @@ def get_jiras_from_python_files(issue_pattern: str, jira_url: str) -> Dict[str, Note: any line containing would be not be checked for presence of a jira id """ - jira_found: Dict[str, list[str]] = {} + jira_found: Dict[str, Set[str]] = {} for filename in all_python_files(): + unique_jiras = set() with open(filename) as fd: # if appears in a line, exclude that line from jira check if unique_jiras := get_jira_ids_from_file_content( @@ -66,6 +67,42 @@ def get_jiras_from_python_files(issue_pattern: str, jira_url: str) -> Dict[str, return jira_found +def get_jira_information( + jira_object: JIRA, + jira_id: str, + skip_project_ids: List[str], + resolved_status: List[str], + jira_target_versions: List[str], + target_version_str: str, +) -> str: + jira_error_string = "" + try: + # check resolved status: + jira_issue_metadata = get_issue(jira=jira_object, jira_id=jira_id).fields + current_jira_status = jira_issue_metadata.status.name.lower() + if current_jira_status in resolved_status: + jira_error_string += f"{jira_id} current status: {current_jira_status} is resolved." + # validate correct target version if provided: + if jira_target_versions: + if skip_project_ids and jira_id.startswith(tuple(skip_project_ids)): + return jira_error_string + fix_version = ( + re.search(r"([\d.]+)", jira_issue_metadata.fixVersions[0].name) + if (jira_issue_metadata.fixVersions) + else None + ) + current_target_version = fix_version.group(1) if fix_version else target_version_str + if not any([current_target_version == version for version in jira_target_versions]): + jira_error_string += ( + f"{jira_id} target version: {current_target_version}, does not match expected " + f"version {jira_target_versions}." + ) + except JIRAError as exp: + jira_error_string += f"{jira_id} JiraError status code: {exp.status_code}, details: {exp.text}]." + + return jira_error_string + + class JiraInvalidConfigFileError(Exception): pass diff --git a/poetry.lock b/poetry.lock index 398cccb..264ce07 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1318,4 +1318,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "84c4492983f778d98a53af09d76bb982052f9a914f20b0f5388fa1214d609408" +content-hash = "68622f58f6cefcaf6d39f9e9c979ccf40b02f48634ea2e97986399ace4fc787f" diff --git a/pyproject.toml b/pyproject.toml index 3e0322f..f054490 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ omit = ["tests/*"] [tool.coverage.report] -fail_under = 60 +fail_under = 40 skip_empty = true [tool.coverage.html] diff --git a/tests/jira_utils/__init__.py b/tests/jira_utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/jira_utils/test_jira.py b/tests/jira_utils/test_jira.py deleted file mode 100644 index e92a42d..0000000 --- a/tests/jira_utils/test_jira.py +++ /dev/null @@ -1,21 +0,0 @@ -import shlex -import subprocess - -from pyhelper_utils.shell import run_command - -from simple_logger.logger import get_logger - -LOGGER = get_logger(name=__name__) -BASE_COMMAND = "poetry run python apps/jira_utils/get_jira_information.py --verbose" - - -def test_jira_missing_config_file(): - rc, _, err = run_command( - command=shlex.split(f"{BASE_COMMAND} --jira-cfg-file invalid-jira.cfg"), - verify_stderr=False, - check=False, - capture_output=False, - stderr=subprocess.PIPE, - ) - assert "Jira config file must contain valid url and token" in err - assert not rc From 52c77517cba7ff970ef22fc03a0213c9e8b906de Mon Sep 17 00:00:00 2001 From: dbasunag Date: Fri, 6 Sep 2024 12:07:32 -0400 Subject: [PATCH 12/17] updates based on review comments --- apps/jira_utils/README.md | 15 ++--- apps/jira_utils/get_jira_information.py | 85 ++++++++++++------------- apps/jira_utils/jira_utils.py | 46 ++++++++++--- apps/utils.py | 2 +- 4 files changed, 86 insertions(+), 62 deletions(-) diff --git a/apps/jira_utils/README.md b/apps/jira_utils/README.md index ddb0367..b1c4b10 100644 --- a/apps/jira_utils/README.md +++ b/apps/jira_utils/README.md @@ -1,7 +1,7 @@ # pyutils-jira Helper to identify wrong jira card references in a repository files. e.g. closed tickets or tickets with not relevant target versions in a test repository branch would be identified. It looks for following patterns in files of a pytest repository: -```bash +```text - jira_id=ABC-12345 # When jira id is present in a function call - /browse/ABC-12345 # when jira is in a link in comments - pytest.mark.jira_utils(ABC-12345) # when jira is in a marker @@ -15,8 +15,7 @@ pyutils-jira --help ``` ## Config file -A config file with the jira connection parameters like url, token, resolved_statuses, skip_project_ids, jira_target_versions should be passed to command line option `--jira-cfg-file` -By default config file location `~/.config/python-utility-scripts/jira_utils/config.cfg` is used, but the location can be overwritten by using the `--jira-cfg-file` option. +A config file with the jira connection parameters like url, token, resolved_statuses, skip_project_ids, target_versions should be passed to command line option `--cfg-file` ### Example: @@ -31,7 +30,7 @@ pyutils-jira: skip_project_ids: - ABC - DEF - jira_target_versions: + target_versions: - 1.0.0 - 2.0.1 issue_pattern: "([A-Z]+-[0-9]+)" @@ -40,21 +39,21 @@ pyutils-jira: This would skip version checks on any jira ids associated with project ABC and DEF This would also check if the current repository branch is pointing to any jira card that is not targeted for 1.0.0 or 2.0.1 version -To run from CLI with `--jira-target-versions` +To run from CLI with `--target-versions` ```bash -pyutils-jira --jira-target-versions '1.0.0,2.0.1' +pyutils-jira --target-versions '1.0.0,2.0.1' ``` #### Note: To mark to skip a jira from these checks, one can add `` as a comment to the same line containing the specific jira Example: -```bash +```text #https://issues.redhat.com/browse/OSD-5716 ``` If resolved_statuses is not provided in config file, by default the following status would be considered as resolved statuses: -```bash +```text ["verified", "release pending", "closed", "resolved"] ``` diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/get_jira_information.py index fca0641..482d299 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/get_jira_information.py @@ -9,12 +9,12 @@ from apps.jira_utils.jira_utils import ( get_jiras_from_python_files, - JiraInvalidConfigFileError, JiraValidationError, get_jira_information, + process_jira_command_line_config_file, ) -from apps.utils import ListParamType, get_util_config +from apps.utils import ListParamType from typing import Dict, List @@ -23,33 +23,32 @@ @click.command() @click.option( - "--jira-cfg-file", + "--cfg-file", help="Provide absolute path to the jira_utils config file. ", type=click.Path(exists=True), - default=os.path.expanduser("~/.config/python-utility-scripts/jira_utils/config.cfg"), ) @click.option( - "--jira-target-versions", + "--target-versions", help="Provide comma separated list of Jira target version, for version validation against a repo branch.", type=ListParamType(), required=False, ) @click.option( - "--jira-skip-projects", + "--skip-projects", help="Provide comma separated list of Jira Project keys, against which version check should be skipped.", type=ListParamType(), ) -@click.option("--jira-url", help="Provide the Jira server URL", type=click.STRING, default=os.getenv("JIRA_SERVER_URL")) -@click.option("--jira-token", help="Provide the Jira token.", type=click.STRING, default=os.getenv("JIRA_TOKEN")) +@click.option("--url", help="Provide the Jira server URL", type=click.STRING, default=os.getenv("JIRA_SERVER_URL")) +@click.option("--token", help="Provide the Jira token.", type=click.STRING, default=os.getenv("JIRA_TOKEN")) @click.option( - "--jira-issue-pattern", + "--issue-pattern", help="Provide the regex for Jira ids", type=click.STRING, show_default=True, default="([A-Z]+-[0-9]+)", ) @click.option( - "--jira-resolved-statuses", + "--resolved-statuses", help="Comma separated list of Jira resolved statuses", type=ListParamType(), show_default=True, @@ -64,13 +63,13 @@ ) @click.option("--verbose", default=False, is_flag=True) def get_jira_mismatch( - jira_cfg_file: str, - jira_target_versions: List[str], - jira_url: str, - jira_token: str, - jira_skip_projects: List[str], - jira_resolved_statuses: List[str], - jira_issue_pattern: str, + cfg_file: str, + target_versions: List[str], + url: str, + token: str, + skip_projects: List[str], + resolved_statuses: List[str], + issue_pattern: str, version_string_not_targeted_jiras: str, verbose: bool, ) -> None: @@ -78,51 +77,47 @@ def get_jira_mismatch( LOGGER.setLevel(logging.DEBUG) else: logging.disable(logging.CRITICAL) - jira_mismatch: Dict[str, str] = {} # Process all the arguments passed from command line or config file or environment variable - config_dict = get_util_config(util_name="pyutils-jira", config_file_path=jira_cfg_file) - jira_url = jira_url or config_dict.get("url", "") - jira_token = jira_token or config_dict.get("token", "") - if not (jira_url and jira_token): - raise JiraInvalidConfigFileError("Jira config file must contain valid url or token.") - - jira_issue_pattern = jira_issue_pattern or config_dict.get("issue_pattern", "") - resolved_status = jira_resolved_statuses or config_dict.get("resolved_statuses", []) - not_targeted_version_str = config_dict.get("version_string_not_targeted_jiras", version_string_not_targeted_jiras) - jira_target_versions = jira_target_versions or config_dict.get("jira_target_versions", []) - skip_project_ids = jira_skip_projects or config_dict.get("skip_project_ids", []) - - jira_obj = JIRA(token_auth=jira_token, options={"server": jira_url}) + jira_config_dict = process_jira_command_line_config_file( + cfg_file=cfg_file, + url=url, + token=token, + resolved_statuses=resolved_statuses, + issue_pattern=issue_pattern, + skip_projects=skip_projects, + version_string_not_targeted_jiras=version_string_not_targeted_jiras, + target_versions=target_versions, + ) + + jira_obj = JIRA(token_auth=jira_config_dict["token"], options={"server": jira_config_dict["url"]}) jira_error: Dict[str, str] = {} - if jira_id_dict := get_jiras_from_python_files(issue_pattern=jira_issue_pattern, jira_url=jira_url): - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + if jira_id_dict := get_jiras_from_python_files( + issue_pattern=jira_config_dict["issue_pattern"], jira_url=jira_config_dict["url"] + ): + with concurrent.futures.ThreadPoolExecutor() as executor: future_to_jiras = { executor.submit( get_jira_information, jira_object=jira_obj, jira_id=jira_id, - skip_project_ids=skip_project_ids, - resolved_status=resolved_status, - jira_target_versions=jira_target_versions, - target_version_str=not_targeted_version_str, + skip_project_ids=jira_config_dict["skip_project_ids"], + resolved_status=jira_config_dict["resolved_status"], + jira_target_versions=jira_config_dict["target_versions"], + target_version_str=jira_config_dict["not_targeted_version_str"], ): jira_id for jira_id in set.union(*jira_id_dict.values()) } for future in concurrent.futures.as_completed(future_to_jiras): jira_id = future_to_jiras[future] + file_names = [file_name for file_name, jiras in jira_id_dict.items() if jira_id in jiras] jira_error_string = future.result() if jira_error_string: - jira_error[jira_id] = jira_error_string - - for file_name, jiras in jira_id_dict.items(): - for jira_id in jiras: - if jira_error.get(jira_id): - jira_mismatch[file_name] = jira_error[jira_id] + jira_error[jira_id] = f" {file_names}: {jira_error_string}" - if jira_mismatch: - error = f"Following Jira ids failed jira check: {jira_mismatch}\n" + if jira_error: + error = f"Following Jira ids failed jira check: {jira_error}\n" LOGGER.error(error) raise JiraValidationError(error) LOGGER.info("Successfully completed Jira validations") diff --git a/apps/jira_utils/jira_utils.py b/apps/jira_utils/jira_utils.py index 70e36db..4d8dbb2 100644 --- a/apps/jira_utils/jira_utils.py +++ b/apps/jira_utils/jira_utils.py @@ -2,12 +2,20 @@ from simple_logger.logger import get_logger from jira import JIRA, JIRAError, Issue from tenacity import retry, retry_if_exception_type, wait_fixed, stop_after_attempt -from apps.utils import all_python_files -from typing import Dict, Set, List +from apps.utils import all_python_files, get_util_config +from typing import Dict, Set, List, Any LOGGER = get_logger(name=__name__) +class JiraInvalidConfigFileError(Exception): + pass + + +class JiraValidationError(Exception): + pass + + @retry(retry=retry_if_exception_type(JIRAError), stop=stop_after_attempt(3), wait=wait_fixed(2)) def get_issue(jira: JIRA, jira_id: str) -> Issue: LOGGER.info(f"Retry staistics for {jira_id}: {get_issue.statistics}") @@ -103,9 +111,31 @@ def get_jira_information( return jira_error_string -class JiraInvalidConfigFileError(Exception): - pass - - -class JiraValidationError(Exception): - pass +def process_jira_command_line_config_file( + cfg_file: str, + url: str, + token: str, + issue_pattern: str, + resolved_statuses: List[str], + version_string_not_targeted_jiras: str, + target_versions: List[str], + skip_projects: List[str], +) -> Dict[str, Any]: + # Process all the arguments passed from command line or config file or environment variable + config_dict = get_util_config(util_name="pyutils-jira", config_file_path=cfg_file) + url = url or config_dict.get("url", "") + token = token or config_dict.get("token", "") + if not (url and token): + raise JiraInvalidConfigFileError("Jira config file must contain valid url or token.") + + return { + "url": url, + "token": token, + "issue_pattern": issue_pattern or config_dict.get("issue_pattern", ""), + "resolved_status": resolved_statuses or config_dict.get("resolved_statuses", []), + "not_targeted_version_str": config_dict.get( + "version_string_not_targeted_jiras", version_string_not_targeted_jiras + ), + "target_versions": target_versions or config_dict.get("target_versions", []), + "skip_project_ids": skip_projects or config_dict.get("skip_project_ids", []), + } diff --git a/apps/utils.py b/apps/utils.py index 852598b..61825b7 100644 --- a/apps/utils.py +++ b/apps/utils.py @@ -11,7 +11,7 @@ def get_util_config(util_name: str, config_file_path: str) -> Dict[str, Any]: - if os.path.exists(config_file_path): + if config_file_path and os.path.exists(config_file_path): with open(config_file_path) as _file: return yaml.safe_load(_file).get(util_name, {}) return {} From d76ab99fb2469902fb85e147361edd1a255efa88 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Wed, 11 Sep 2024 13:48:53 +0300 Subject: [PATCH 13/17] refacot code --- ...ira_information.py => jira_information.py} | 66 +++++++++---------- apps/jira_utils/jira_utils.py | 37 ++++++----- pyproject.toml | 2 +- 3 files changed, 54 insertions(+), 51 deletions(-) rename apps/jira_utils/{get_jira_information.py => jira_information.py} (64%) diff --git a/apps/jira_utils/get_jira_information.py b/apps/jira_utils/jira_information.py similarity index 64% rename from apps/jira_utils/get_jira_information.py rename to apps/jira_utils/jira_information.py index 482d299..f288e07 100644 --- a/apps/jira_utils/get_jira_information.py +++ b/apps/jira_utils/jira_information.py @@ -1,4 +1,5 @@ import logging +import sys import os import concurrent.futures @@ -9,7 +10,6 @@ from apps.jira_utils.jira_utils import ( get_jiras_from_python_files, - JiraValidationError, get_jira_information, process_jira_command_line_config_file, ) @@ -23,15 +23,14 @@ @click.command() @click.option( - "--cfg-file", - help="Provide absolute path to the jira_utils config file. ", + "--config", + help="Provide absolute path to the jira_utils config file.", type=click.Path(exists=True), ) @click.option( "--target-versions", help="Provide comma separated list of Jira target version, for version validation against a repo branch.", type=ListParamType(), - required=False, ) @click.option( "--skip-projects", @@ -63,7 +62,7 @@ ) @click.option("--verbose", default=False, is_flag=True) def get_jira_mismatch( - cfg_file: str, + config: str, target_versions: List[str], url: str, token: str, @@ -73,13 +72,15 @@ def get_jira_mismatch( version_string_not_targeted_jiras: str, verbose: bool, ) -> None: - if verbose: - LOGGER.setLevel(logging.DEBUG) - else: - logging.disable(logging.CRITICAL) + LOGGER.setLevel(logging.DEBUG if verbose else logging.INFO) + + if not (config or token or url): + LOGGER.error("Config file or token or url is required") + sys.exit(1) + # Process all the arguments passed from command line or config file or environment variable jira_config_dict = process_jira_command_line_config_file( - cfg_file=cfg_file, + config=config, url=url, token=token, resolved_statuses=resolved_statuses, @@ -96,31 +97,30 @@ def get_jira_mismatch( issue_pattern=jira_config_dict["issue_pattern"], jira_url=jira_config_dict["url"] ): with concurrent.futures.ThreadPoolExecutor() as executor: - future_to_jiras = { - executor.submit( - get_jira_information, - jira_object=jira_obj, - jira_id=jira_id, - skip_project_ids=jira_config_dict["skip_project_ids"], - resolved_status=jira_config_dict["resolved_status"], - jira_target_versions=jira_config_dict["target_versions"], - target_version_str=jira_config_dict["not_targeted_version_str"], - ): jira_id - for jira_id in set.union(*jira_id_dict.values()) - } - - for future in concurrent.futures.as_completed(future_to_jiras): - jira_id = future_to_jiras[future] - file_names = [file_name for file_name, jiras in jira_id_dict.items() if jira_id in jiras] - jira_error_string = future.result() - if jira_error_string: - jira_error[jira_id] = f" {file_names}: {jira_error_string}" + for file_name, ids in jira_id_dict.items(): + for jira_id in ids: + future_to_jiras = { + executor.submit( + get_jira_information, + jira_object=jira_obj, + jira_id=jira_id, + skip_project_ids=jira_config_dict["skip_project_ids"], + resolved_status=jira_config_dict["resolved_status"], + jira_target_versions=jira_config_dict["target_versions"], + target_version_str=jira_config_dict["not_targeted_version_str"], + file_name=file_name, + ) + } + + for future in concurrent.futures.as_completed(future_to_jiras): + file_name, jira_error_string = future.result() + if jira_error_string: + jira_error[file_name] = jira_error_string if jira_error: - error = f"Following Jira ids failed jira check: {jira_error}\n" - LOGGER.error(error) - raise JiraValidationError(error) - LOGGER.info("Successfully completed Jira validations") + _jira_error = "\n\t".join([f"{key}: {val}" for key, val in jira_error.items()]) + LOGGER.error(f"Following Jira ids failed jira check: \n\t{_jira_error}\n") + sys.exit(1) if __name__ == "__main__": diff --git a/apps/jira_utils/jira_utils.py b/apps/jira_utils/jira_utils.py index 4d8dbb2..f02edcd 100644 --- a/apps/jira_utils/jira_utils.py +++ b/apps/jira_utils/jira_utils.py @@ -1,24 +1,19 @@ +from functools import lru_cache +import sys import re from simple_logger.logger import get_logger from jira import JIRA, JIRAError, Issue from tenacity import retry, retry_if_exception_type, wait_fixed, stop_after_attempt from apps.utils import all_python_files, get_util_config -from typing import Dict, Set, List, Any +from typing import Dict, Set, List, Any, Tuple LOGGER = get_logger(name=__name__) -class JiraInvalidConfigFileError(Exception): - pass - - -class JiraValidationError(Exception): - pass - - +@lru_cache @retry(retry=retry_if_exception_type(JIRAError), stop=stop_after_attempt(3), wait=wait_fixed(2)) def get_issue(jira: JIRA, jira_id: str) -> Issue: - LOGGER.info(f"Retry staistics for {jira_id}: {get_issue.statistics}") + LOGGER.debug(f"Retry staistics for {jira_id}: {get_issue.statistics}") return jira.issue(id=jira_id, fields="status, issuetype, fixVersions") @@ -43,6 +38,7 @@ def get_jira_ids_from_file_content(file_content: str, issue_pattern: str, jira_u rf"{jira_url}/browse/{issue_pattern}", file_content, ) + return set(_pytest_jira_marker_bugs + _jira_id_arguments + _jira_url_jiras) @@ -70,8 +66,11 @@ def get_jiras_from_python_files(issue_pattern: str, jira_url: str) -> Dict[str, jira_url=jira_url, ): jira_found[filename] = unique_jiras + if jira_found: - LOGGER.info(f"Following jiras are found: {jira_found}") + _jira_found = "\n\t".join([f"{key}: {val}" for key, val in jira_found.items()]) + LOGGER.debug(f"Following jiras are found: \n\t{_jira_found}") + return jira_found @@ -82,7 +81,8 @@ def get_jira_information( resolved_status: List[str], jira_target_versions: List[str], target_version_str: str, -) -> str: + file_name: str, +) -> Tuple[str, str]: jira_error_string = "" try: # check resolved status: @@ -90,10 +90,12 @@ def get_jira_information( current_jira_status = jira_issue_metadata.status.name.lower() if current_jira_status in resolved_status: jira_error_string += f"{jira_id} current status: {current_jira_status} is resolved." + # validate correct target version if provided: if jira_target_versions: if skip_project_ids and jira_id.startswith(tuple(skip_project_ids)): - return jira_error_string + return file_name, jira_error_string + fix_version = ( re.search(r"([\d.]+)", jira_issue_metadata.fixVersions[0].name) if (jira_issue_metadata.fixVersions) @@ -108,11 +110,11 @@ def get_jira_information( except JIRAError as exp: jira_error_string += f"{jira_id} JiraError status code: {exp.status_code}, details: {exp.text}]." - return jira_error_string + return file_name, jira_error_string def process_jira_command_line_config_file( - cfg_file: str, + config: str, url: str, token: str, issue_pattern: str, @@ -122,11 +124,12 @@ def process_jira_command_line_config_file( skip_projects: List[str], ) -> Dict[str, Any]: # Process all the arguments passed from command line or config file or environment variable - config_dict = get_util_config(util_name="pyutils-jira", config_file_path=cfg_file) + config_dict = get_util_config(util_name="pyutils-jira", config_file_path=config) url = url or config_dict.get("url", "") token = token or config_dict.get("token", "") if not (url and token): - raise JiraInvalidConfigFileError("Jira config file must contain valid url or token.") + LOGGER.error("Jira config file must contain valid url or token.") + sys.exit(1) return { "url": url, diff --git a/pyproject.toml b/pyproject.toml index f054490..1b2e82b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ packages = [{ include = "apps" }] pyutils-unusedcode = "apps.unused_code.unused_code:get_unused_functions" pyutils-polarion-verify-tc-requirements = "apps.polarion.polarion_verify_tc_requirements:has_verify" pyutils-polarion-set-automated = "apps.polarion.polarion_set_automated:polarion_approve_automate" -pyutils-jira = "apps.jira_utils.get_jira_information:get_jira_mismatch" +pyutils-jira = "apps.jira_utils.jira_information:get_jira_mismatch" [tool.poetry.dependencies] From afa7f35f7b63d7935bf8eaf698b8b8b96025eb60 Mon Sep 17 00:00:00 2001 From: dbasunag Date: Wed, 11 Sep 2024 19:25:07 -0400 Subject: [PATCH 14/17] minor adjustments --- apps/jira_utils/jira_information.py | 165 +++++++++++++++++++++++++--- apps/jira_utils/jira_utils.py | 144 ------------------------ 2 files changed, 150 insertions(+), 159 deletions(-) delete mode 100644 apps/jira_utils/jira_utils.py diff --git a/apps/jira_utils/jira_information.py b/apps/jira_utils/jira_information.py index f288e07..e581c3a 100644 --- a/apps/jira_utils/jira_information.py +++ b/apps/jira_utils/jira_information.py @@ -1,29 +1,165 @@ import logging +import re import sys import os import concurrent.futures +from functools import lru_cache import click -from jira import JIRA +from jira import JIRA, JIRAError, Issue from simple_logger.logger import get_logger -from apps.jira_utils.jira_utils import ( - get_jiras_from_python_files, - get_jira_information, - process_jira_command_line_config_file, -) - from apps.utils import ListParamType -from typing import Dict, List +from typing import Dict, List, Set, Tuple, Any +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed +from apps.utils import all_python_files, get_util_config LOGGER = get_logger(name=__name__) +@retry(retry=retry_if_exception_type(JIRAError), stop=stop_after_attempt(3), wait=wait_fixed(2)) +@lru_cache +def get_issue( + jira: JIRA, + jira_id: str, +) -> Issue: + LOGGER.debug(f"Retry staistics for {jira_id}: {get_issue.statistics}") + return jira.issue(id=jira_id, fields="status, issuetype, fixVersions") + + +def get_jira_ids_from_file_content(file_content: str, issue_pattern: str, jira_url: str) -> Set[str]: + """ + Try to find all Jira tickets in a given file content. + Looking for the following patterns: + - jira_id=ABC-12345 # When jira id is present in a function call + - /browse/ABC-12345 # when jira is in a link in comments + - pytest.mark.jira_utils(ABC-12345) # when jira is in a marker + + Args: + file_content (str): The content of a given file. + issue_pattern (str): regex pattern for jira ids + + Returns: + set: A set of jira tickets. + """ + _pytest_jira_marker_bugs = re.findall(rf"pytest.mark.jira.*?{issue_pattern}.*", file_content, re.DOTALL) + _jira_id_arguments = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content) + _jira_url_jiras = re.findall( + rf"{jira_url}/browse/{issue_pattern}", + file_content, + ) + + return set(_pytest_jira_marker_bugs + _jira_id_arguments + _jira_url_jiras) + + +def get_jiras_from_python_files(issue_pattern: str, jira_url: str) -> Dict[str, Set[str]]: + """ + Get all python files from the current directory and get list of jira ids from each of them + + Args: + issue_pattern (str): regex pattern for jira ids + jira_url (str): jira url that could be used to look for possible presence of jira references in a file + + Returns: + Dict: A dict of filenames and associated jira tickets. + + Note: any line containing would be not be checked for presence of a jira id + """ + jira_found: Dict[str, Set[str]] = {} + for filename in all_python_files(): + unique_jiras = set() + with open(filename) as fd: + # if appears in a line, exclude that line from jira check + if unique_jiras := get_jira_ids_from_file_content( + file_content="\n".join([line for line in fd.readlines() if "" not in line]), + issue_pattern=issue_pattern, + jira_url=jira_url, + ): + jira_found[filename] = unique_jiras + + if jira_found: + _jira_found = "\n\t".join([f"{key}: {val}" for key, val in jira_found.items()]) + LOGGER.debug(f"Following jiras are found: \n\t{_jira_found}") + + return jira_found + + +def get_jira_information( + jira_object: JIRA, + jira_id: str, + skip_project_ids: List[str], + resolved_status: List[str], + jira_target_versions: List[str], + target_version_str: str, + file_name: str, +) -> Tuple[str, str]: + jira_error_string = "" + try: + # check resolved status: + jira_issue_metadata = get_issue(jira=jira_object, jira_id=jira_id).fields + current_jira_status = jira_issue_metadata.status.name.lower() + LOGGER.debug(f"Jira: {jira_id}, status: {current_jira_status}") + if current_jira_status in resolved_status: + jira_error_string += f"{jira_id} current status: {current_jira_status} is resolved." + + # validate correct target version if provided: + if jira_target_versions: + if skip_project_ids and jira_id.startswith(tuple(skip_project_ids)): + return file_name, jira_error_string + + fix_version = ( + re.search(r"([\d.]+)", jira_issue_metadata.fixVersions[0].name) + if (jira_issue_metadata.fixVersions) + else None + ) + current_target_version = fix_version.group(1) if fix_version else target_version_str + if not any([current_target_version == version for version in jira_target_versions]): + jira_error_string += ( + f"{jira_id} target version: {current_target_version}, does not match expected " + f"version {jira_target_versions}." + ) + except JIRAError as exp: + jira_error_string += f"{jira_id} JiraError status code: {exp.status_code}, details: {exp.text}]." + + return file_name, jira_error_string + + +def process_jira_command_line_config_file( + config_file_path: str, + url: str, + token: str, + issue_pattern: str, + resolved_statuses: List[str], + version_string_not_targeted_jiras: str, + target_versions: List[str], + skip_projects: List[str], +) -> Dict[str, Any]: + # Process all the arguments passed from command line or config file or environment variable + config_dict = get_util_config(util_name="pyutils-jira", config_file_path=config_file_path) + url = url or config_dict.get("url", "") + token = token or config_dict.get("token", "") + if not (url and token): + LOGGER.error("Jira config file must contain valid url or token.") + sys.exit(1) + + return { + "url": url, + "token": token, + "issue_pattern": issue_pattern or config_dict.get("issue_pattern", ""), + "resolved_status": resolved_statuses or config_dict.get("resolved_statuses", []), + "not_targeted_version_str": config_dict.get( + "version_string_not_targeted_jiras", version_string_not_targeted_jiras + ), + "target_versions": target_versions or config_dict.get("target_versions", []), + "skip_project_ids": skip_projects or config_dict.get("skip_project_ids", []), + } + + @click.command() @click.option( - "--config", + "--config-file-path", help="Provide absolute path to the jira_utils config file.", type=click.Path(exists=True), ) @@ -62,7 +198,7 @@ ) @click.option("--verbose", default=False, is_flag=True) def get_jira_mismatch( - config: str, + config_file_path: str, target_versions: List[str], url: str, token: str, @@ -73,14 +209,13 @@ def get_jira_mismatch( verbose: bool, ) -> None: LOGGER.setLevel(logging.DEBUG if verbose else logging.INFO) - - if not (config or token or url): - LOGGER.error("Config file or token or url is required") + if not (config_file_path or (token and url)): + LOGGER.error("Config file or token and url are required.") sys.exit(1) # Process all the arguments passed from command line or config file or environment variable jira_config_dict = process_jira_command_line_config_file( - config=config, + config_file_path=config_file_path, url=url, token=token, resolved_statuses=resolved_statuses, @@ -119,7 +254,7 @@ def get_jira_mismatch( if jira_error: _jira_error = "\n\t".join([f"{key}: {val}" for key, val in jira_error.items()]) - LOGGER.error(f"Following Jira ids failed jira check: \n\t{_jira_error}\n") + LOGGER.error(f"Following Jira ids failed jira version/statuscheck: \n\t{_jira_error}\n") sys.exit(1) diff --git a/apps/jira_utils/jira_utils.py b/apps/jira_utils/jira_utils.py deleted file mode 100644 index f02edcd..0000000 --- a/apps/jira_utils/jira_utils.py +++ /dev/null @@ -1,144 +0,0 @@ -from functools import lru_cache -import sys -import re -from simple_logger.logger import get_logger -from jira import JIRA, JIRAError, Issue -from tenacity import retry, retry_if_exception_type, wait_fixed, stop_after_attempt -from apps.utils import all_python_files, get_util_config -from typing import Dict, Set, List, Any, Tuple - -LOGGER = get_logger(name=__name__) - - -@lru_cache -@retry(retry=retry_if_exception_type(JIRAError), stop=stop_after_attempt(3), wait=wait_fixed(2)) -def get_issue(jira: JIRA, jira_id: str) -> Issue: - LOGGER.debug(f"Retry staistics for {jira_id}: {get_issue.statistics}") - return jira.issue(id=jira_id, fields="status, issuetype, fixVersions") - - -def get_jira_ids_from_file_content(file_content: str, issue_pattern: str, jira_url: str) -> Set[str]: - """ - Try to find all Jira tickets in a given file content. - Looking for the following patterns: - - jira_id=ABC-12345 # When jira id is present in a function call - - /browse/ABC-12345 # when jira is in a link in comments - - pytest.mark.jira_utils(ABC-12345) # when jira is in a marker - - Args: - file_content (str): The content of a given file. - issue_pattern (str): regex pattern for jira ids - - Returns: - set: A set of jira tickets. - """ - _pytest_jira_marker_bugs = re.findall(rf"pytest.mark.jira.*?{issue_pattern}.*", file_content, re.DOTALL) - _jira_id_arguments = re.findall(rf"jira_id\s*=[\s*\"\']*{issue_pattern}.*", file_content) - _jira_url_jiras = re.findall( - rf"{jira_url}/browse/{issue_pattern}", - file_content, - ) - - return set(_pytest_jira_marker_bugs + _jira_id_arguments + _jira_url_jiras) - - -def get_jiras_from_python_files(issue_pattern: str, jira_url: str) -> Dict[str, Set[str]]: - """ - Get all python files from the current directory and get list of jira ids from each of them - - Args: - issue_pattern (str): regex pattern for jira ids - jira_url (str): jira url that could be used to look for possible presence of jira references in a file - - Returns: - Dict: A dict of filenames and associated jira tickets. - - Note: any line containing would be not be checked for presence of a jira id - """ - jira_found: Dict[str, Set[str]] = {} - for filename in all_python_files(): - unique_jiras = set() - with open(filename) as fd: - # if appears in a line, exclude that line from jira check - if unique_jiras := get_jira_ids_from_file_content( - file_content="\n".join([line for line in fd.readlines() if "" not in line]), - issue_pattern=issue_pattern, - jira_url=jira_url, - ): - jira_found[filename] = unique_jiras - - if jira_found: - _jira_found = "\n\t".join([f"{key}: {val}" for key, val in jira_found.items()]) - LOGGER.debug(f"Following jiras are found: \n\t{_jira_found}") - - return jira_found - - -def get_jira_information( - jira_object: JIRA, - jira_id: str, - skip_project_ids: List[str], - resolved_status: List[str], - jira_target_versions: List[str], - target_version_str: str, - file_name: str, -) -> Tuple[str, str]: - jira_error_string = "" - try: - # check resolved status: - jira_issue_metadata = get_issue(jira=jira_object, jira_id=jira_id).fields - current_jira_status = jira_issue_metadata.status.name.lower() - if current_jira_status in resolved_status: - jira_error_string += f"{jira_id} current status: {current_jira_status} is resolved." - - # validate correct target version if provided: - if jira_target_versions: - if skip_project_ids and jira_id.startswith(tuple(skip_project_ids)): - return file_name, jira_error_string - - fix_version = ( - re.search(r"([\d.]+)", jira_issue_metadata.fixVersions[0].name) - if (jira_issue_metadata.fixVersions) - else None - ) - current_target_version = fix_version.group(1) if fix_version else target_version_str - if not any([current_target_version == version for version in jira_target_versions]): - jira_error_string += ( - f"{jira_id} target version: {current_target_version}, does not match expected " - f"version {jira_target_versions}." - ) - except JIRAError as exp: - jira_error_string += f"{jira_id} JiraError status code: {exp.status_code}, details: {exp.text}]." - - return file_name, jira_error_string - - -def process_jira_command_line_config_file( - config: str, - url: str, - token: str, - issue_pattern: str, - resolved_statuses: List[str], - version_string_not_targeted_jiras: str, - target_versions: List[str], - skip_projects: List[str], -) -> Dict[str, Any]: - # Process all the arguments passed from command line or config file or environment variable - config_dict = get_util_config(util_name="pyutils-jira", config_file_path=config) - url = url or config_dict.get("url", "") - token = token or config_dict.get("token", "") - if not (url and token): - LOGGER.error("Jira config file must contain valid url or token.") - sys.exit(1) - - return { - "url": url, - "token": token, - "issue_pattern": issue_pattern or config_dict.get("issue_pattern", ""), - "resolved_status": resolved_statuses or config_dict.get("resolved_statuses", []), - "not_targeted_version_str": config_dict.get( - "version_string_not_targeted_jiras", version_string_not_targeted_jiras - ), - "target_versions": target_versions or config_dict.get("target_versions", []), - "skip_project_ids": skip_projects or config_dict.get("skip_project_ids", []), - } From 5e17df6b4d406d389ab4b7e939cc8ca187741b6a Mon Sep 17 00:00:00 2001 From: dbasunag Date: Tue, 24 Sep 2024 15:49:40 -0400 Subject: [PATCH 15/17] update poetry.lock --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 264ce07..4d81fb1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1318,4 +1318,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "68622f58f6cefcaf6d39f9e9c979ccf40b02f48634ea2e97986399ace4fc787f" +content-hash = "1ceeba9a251043942e466ea8e9f718409c53d3d19c315ca0596e04d9e0676aaa" From ee31de75664fd514e55b0c6dded7535fcd382d6c Mon Sep 17 00:00:00 2001 From: dbasunag Date: Mon, 30 Sep 2024 18:32:33 -0400 Subject: [PATCH 16/17] addressed comments --- apps/jira_utils/jira_information.py | 20 +++++++++++--------- apps/utils.py | 4 ++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/apps/jira_utils/jira_information.py b/apps/jira_utils/jira_information.py index e581c3a..5f13280 100644 --- a/apps/jira_utils/jira_information.py +++ b/apps/jira_utils/jira_information.py @@ -32,6 +32,7 @@ def get_issue( def get_jira_ids_from_file_content(file_content: str, issue_pattern: str, jira_url: str) -> Set[str]: """ Try to find all Jira tickets in a given file content. + Looking for the following patterns: - jira_id=ABC-12345 # When jira id is present in a function call - /browse/ABC-12345 # when jira is in a link in comments @@ -69,15 +70,16 @@ def get_jiras_from_python_files(issue_pattern: str, jira_url: str) -> Dict[str, """ jira_found: Dict[str, Set[str]] = {} for filename in all_python_files(): - unique_jiras = set() + file_content = [] with open(filename) as fd: - # if appears in a line, exclude that line from jira check - if unique_jiras := get_jira_ids_from_file_content( - file_content="\n".join([line for line in fd.readlines() if "" not in line]), - issue_pattern=issue_pattern, - jira_url=jira_url, - ): - jira_found[filename] = unique_jiras + file_content = fd.readlines() + # if appears in a line, exclude that line from jira check + if unique_jiras := get_jira_ids_from_file_content( + file_content="\n".join([line for line in file_content if "" not in line]), + issue_pattern=issue_pattern, + jira_url=jira_url, + ): + jira_found[filename] = unique_jiras if jira_found: _jira_found = "\n\t".join([f"{key}: {val}" for key, val in jira_found.items()]) @@ -141,7 +143,7 @@ def process_jira_command_line_config_file( url = url or config_dict.get("url", "") token = token or config_dict.get("token", "") if not (url and token): - LOGGER.error("Jira config file must contain valid url or token.") + LOGGER.error("Jira url and token are required.") sys.exit(1) return { diff --git a/apps/utils.py b/apps/utils.py index 61825b7..7a4f925 100644 --- a/apps/utils.py +++ b/apps/utils.py @@ -5,12 +5,12 @@ from simple_logger.logger import get_logger import json import click -from typing import Any, Dict, Iterable +from typing import Any, Dict, Iterable, Optional LOGGER = get_logger(name=__name__) -def get_util_config(util_name: str, config_file_path: str) -> Dict[str, Any]: +def get_util_config(util_name: str, config_file_path: Optional[str] = None) -> Dict[str, Any]: if config_file_path and os.path.exists(config_file_path): with open(config_file_path) as _file: return yaml.safe_load(_file).get(util_name, {}) From efc8c476d8028e4ad3b5e607dc7aa326f4e29916 Mon Sep 17 00:00:00 2001 From: dbasunag Date: Tue, 1 Oct 2024 14:29:27 -0400 Subject: [PATCH 17/17] update lock file --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 93d2c96..735a7f9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1318,4 +1318,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "1ceeba9a251043942e466ea8e9f718409c53d3d19c315ca0596e04d9e0676aaa" +content-hash = "d666cbb73412e8a79de21a69f4d749362a65456b24554d5f5fdea9b3e5d0e756"