From 5c72b5a55b114ada55669bc72f1c478473e0797d Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Sat, 1 Feb 2025 15:14:09 -0300 Subject: [PATCH 1/5] add --debug command option So we can push some of messages we print today out for clear view for our audience who only want to see info relevant to their test and tree. Signed-off-by: Gustavo Padovan --- kcidev/libs/common.py | 5 +++++ kcidev/libs/maestro_common.py | 4 ++-- kcidev/main.py | 10 +++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/kcidev/libs/common.py b/kcidev/libs/common.py index 24f00e9..c7cf057 100644 --- a/kcidev/libs/common.py +++ b/kcidev/libs/common.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import logging import os import sys @@ -65,6 +66,10 @@ def config_path(settings): return global_path +def kci_info(content): + logging.info(content) + + def kci_msg(content): click.echo(content) diff --git a/kcidev/libs/maestro_common.py b/kcidev/libs/maestro_common.py index b890f6e..8393e62 100644 --- a/kcidev/libs/maestro_common.py +++ b/kcidev/libs/maestro_common.py @@ -11,9 +11,9 @@ def maestro_print_api_call(host, data=None): - click.secho("maestro api endpoint: " + host, fg="green") + kci_info("maestro api endpoint: " + host) if data: - kci_log(json.dumps(data, indent=4)) + kci_info(json.dumps(data, indent=4)) def maestro_api_error(response): diff --git a/kcidev/main.py b/kcidev/main.py index 84687ee..691b754 100755 --- a/kcidev/main.py +++ b/kcidev/main.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import logging + import click from kcidev.libs.common import * @@ -28,8 +30,14 @@ required=False, ) @click.option("--instance", help="API instance to use", required=False) +@click.option("--debug", is_flag=True, help="Enable debug info") @click.pass_context -def cli(ctx, settings, instance): +def cli(ctx, settings, instance, debug): + if debug: + # DEBUG level is too verbose about included packages + # us INFO instead + logging.basicConfig(level=logging.INFO) + subcommand = ctx.invoked_subcommand ctx.obj = {"CFG": load_toml(settings, subcommand)} ctx.obj["SETTINGS"] = settings From 9e0333eb5cfccb2837651f105d0618d3230b8b83 Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Sat, 1 Feb 2025 16:15:00 -0300 Subject: [PATCH 2/5] watch: walk back from using Click.Abort() Abort won't exit immedially, but generate an Click execption instead. However, we want to exit right away. The suggestion is to just use sys.exit() Signed-off-by: Gustavo Padovan --- kcidev/libs/maestro_common.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kcidev/libs/maestro_common.py b/kcidev/libs/maestro_common.py index 8393e62..04c2f24 100644 --- a/kcidev/libs/maestro_common.py +++ b/kcidev/libs/maestro_common.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import errno import json import time @@ -53,10 +54,10 @@ def maestro_get_node(url, nodeid): response.raise_for_status() except requests.exceptions.HTTPError as ex: kci_err(ex.response.json().get("detail")) - click.Abort() + sys.exit(errno.ENOENT) except Exception as ex: kci_err(ex) - click.Abort() + sys.exit(errno.ENOENT) return response.json() @@ -78,10 +79,10 @@ def maestro_get_nodes(url, limit, offset, filter): response.raise_for_status() except requests.exceptions.HTTPError as ex: kci_err(ex.response.json().get("detail")) - click.Abort() + sys.exit(errno.ENOENT) except Exception as ex: kci_err(ex) - click.Abort() + sys.exit(errno.ENOENT) return response.json() From 5d3e46083646d5176cfaf941e8462edea8caa633 Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Mon, 3 Feb 2025 07:31:32 -0300 Subject: [PATCH 3/5] checkout: Improve checkout command description Signed-off-by: Gustavo Padovan --- kcidev/subcommands/checkout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kcidev/subcommands/checkout.py b/kcidev/subcommands/checkout.py index 5ceb51b..5566b4a 100644 --- a/kcidev/subcommands/checkout.py +++ b/kcidev/subcommands/checkout.py @@ -59,7 +59,7 @@ def retrieve_tot_commit(repourl, branch): return sha -@click.command(help="Create custom tree checkout on KernelCI and trigger a tests") +@click.command(help="Trigger a test for a specific tree/branch/commit") @click.option( "--giturl", help="Git URL to checkout", From dda56521af2b28d51e00765b77ec164a70731ed1 Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Sun, 2 Feb 2025 09:18:28 -0300 Subject: [PATCH 4/5] watch: make output clear and concise Our users don't need to see the background infomation from Maestro. We can show only the needful information. With time we will also show links to logs and other info in concise way. Signed-off-by: Gustavo Padovan --- kcidev/libs/maestro_common.py | 78 +++++++++++++++++++++++++---------- kcidev/subcommands/watch.py | 2 +- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/kcidev/libs/maestro_common.py b/kcidev/libs/maestro_common.py index 04c2f24..d1de285 100644 --- a/kcidev/libs/maestro_common.py +++ b/kcidev/libs/maestro_common.py @@ -107,7 +107,7 @@ def maestro_check_node(node): else: if state == "running": return "RUNNING" - elif state == "done" and result == "pass": + elif state == "done" and (result == "pass" or result == "fail"): return "DONE" else: return "FAIL" @@ -135,17 +135,57 @@ def maestro_retrieve_treeid_nodes(baseurl, token, treeid): return response.json() +def maestro_node_result(node): + result = node["result"] + if node["kind"] == "checkout": + if ( + result == "available" + or result == "closing" + or result == "done" + or result == "pass" + ): + kci_msg_green_nonl("PASS") + elif result == None: + kci_msg_green_nonl("PASS") + else: + kci_msg_red_nonl("FAIL") + else: + if node["result"] == "pass": + kci_msg_green_nonl("PASS") + elif node["result"] == "fail": + kci_msg_red_nonl("FAIL") + else: + kci_msg_yellow_nonl(node["result"]) + + if node["kind"] == "checkout": + kci_msg_nonl(" branch checkout") + else: + kci_msg_nonl(f" {node["kind"]}: ") + + if node["kind"] != "checkout": + kci_msg_nonl(f"{node["name"]}") + + kci_msg(f" - node_id:{node["id"]} ({node["updated"]})") + + def maestro_watch_jobs(baseurl, token, treeid, job_filter, test): # we need to add to job_filter "checkout" node job_filter = list(job_filter) job_filter.append("checkout") + kci_log(f"job_filter: {", ".join(job_filter)}") previous_nodes = None + running = False + + job_info = {} + for job in job_filter: + job_info[job] = {"done": False, "running": False} + while True: inprogress = 0 joblist = job_filter.copy() nodes = maestro_retrieve_treeid_nodes(baseurl, token, treeid) if not nodes: - click.secho("No nodes found. Retrying...", fg="yellow") + kci_warning("No nodes found. Retrying...") time.sleep(5) continue if previous_nodes == nodes: @@ -154,10 +194,7 @@ def maestro_watch_jobs(baseurl, token, treeid, job_filter, test): continue time_local = time.localtime() - click.echo(f"\nCurrent time: {time.strftime('%Y-%m-%d %H:%M:%S', time_local)}") - click.secho( - f"Total tree nodes {len(nodes)} found. job_filter: {job_filter}", fg="green" - ) + kci_info(f"\nCurrent time: {time.strftime('%Y-%m-%d %H:%M:%S', time_local)}") # Tricky part in watch is that we might have one item in job_filter (job, test), # but it might spawn multiple nodes with same name @@ -167,30 +204,29 @@ def maestro_watch_jobs(baseurl, token, treeid, job_filter, test): if node["name"] == test: test_result = node["result"] if node["name"] in job_filter: - result = maestro_check_node(node) - if result == "DONE": + status = maestro_check_node(node) + if status == "DONE": + if job_info[node["name"]]["running"]: + kci_msg("") + job_info[node["name"]]["running"] = False + if not job_info[node["name"]]["done"]: + maestro_node_result(node) + job_info[node["name"]]["done"] = True if isinstance(joblist, list) and node["name"] in joblist: joblist.remove(node["name"]) - color = "green" - elif result == "RUNNING": + elif status == "RUNNING": + job_info[node["name"]]["running"] = True inprogress += 1 - color = "yellow" else: if isinstance(joblist, list) and node["name"] in joblist: joblist.remove(node["name"]) - color = "red" # if test is same as job, dont indicate infra-failure if test job fail if test and test != node["name"]: # if we have a test, and prior job failed, we should indicate that kci_err(f"Job {node['name']} failed, test can't be executed") sys.exit(2) - nodeid = node.get("id") - click.secho( - f"Node: {nodeid} job: {node['name']} State: {node['state']} Result: {node['result']}", - fg=color, - ) if isinstance(joblist, list) and len(joblist) == 0 and inprogress == 0: - click.secho("All jobs completed", fg="green") + kci_info("All jobs completed") if not test: return else: @@ -202,13 +238,11 @@ def maestro_watch_jobs(baseurl, token, treeid, job_filter, test): continue if test_result and test_result == "pass": - click.secho(f"Test {test} passed", fg="green") sys.exit(0) elif test_result: - # ignore null, that means result not ready yet - kci_err(f"Test {test} failed: {test_result}") sys.exit(1) - kci_msg_nonl(f"\rRefresh every 30s...") + running = True + kci_msg_nonl(f"\rRunning job...") previous_nodes = nodes time.sleep(30) diff --git a/kcidev/subcommands/watch.py b/kcidev/subcommands/watch.py index c63be97..7e46c02 100644 --- a/kcidev/subcommands/watch.py +++ b/kcidev/subcommands/watch.py @@ -20,7 +20,7 @@ ) @click.option( "--test", - help="Return code based on the test result", + help="Return 0 if the test name supplied passed, 1 otherwise", ) @click.pass_context def watch(ctx, nodeid, job_filter, test): From 4d871146908216bca8b18aca39efe6eab88704bf Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Tue, 4 Feb 2025 08:12:17 -0300 Subject: [PATCH 5/5] watch: show error message and exit if node not found Signed-off-by: Gustavo Padovan --- kcidev/subcommands/watch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kcidev/subcommands/watch.py b/kcidev/subcommands/watch.py index 7e46c02..338ae5f 100644 --- a/kcidev/subcommands/watch.py +++ b/kcidev/subcommands/watch.py @@ -31,6 +31,9 @@ def watch(ctx, nodeid, job_filter, test): token = cfg[instance]["token"] node = maestro_get_node(apiurl, nodeid) + if not node: + kci_err(f"node id {nodeid} not found.") + sys.exit(errno.ENOENT) maestro_watch_jobs(apiurl, token, node["treeid"], job_filter, test)