From 9325a5e55184558626329ad61245c1028f10af3e Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Thu, 16 Jan 2025 10:06:45 -0300 Subject: [PATCH 1/7] checkout: improve output of job status message Add colon punctuation to separate items better. Signed-off-by: Gustavo Padovan --- kcidev/subcommands/checkout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kcidev/subcommands/checkout.py b/kcidev/subcommands/checkout.py index 6bc6a9c..4fa255f 100644 --- a/kcidev/subcommands/checkout.py +++ b/kcidev/subcommands/checkout.py @@ -152,7 +152,7 @@ def watch_jobs(baseurl, token, treeid, job_filter, test): sys.exit(2) nodeid = node.get("id") click.secho( - f"Node {nodeid} job {node['name']} State {node['state']} Result {node['result']}", + f"Node: {nodeid} job: {node['name']} State: {node['state']} Result: {node['result']}", fg=color, ) if len(joblist) == 0 and inprogress == 0: From 491dfc902796c7042c655aecbff262151c8d5ae4 Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Thu, 16 Jan 2025 10:20:35 -0300 Subject: [PATCH 2/7] checkout: reduce noise by removing duplicated output The checkout command in printing the same information multiple times before the nodes change again. Let's print it once. Signed-off-by: Gustavo Padovan --- kcidev/subcommands/checkout.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kcidev/subcommands/checkout.py b/kcidev/subcommands/checkout.py index 4fa255f..efadd87 100644 --- a/kcidev/subcommands/checkout.py +++ b/kcidev/subcommands/checkout.py @@ -111,6 +111,7 @@ def watch_jobs(baseurl, token, treeid, job_filter, test): # we need to add to job_filter "checkout" node job_filter = list(job_filter) job_filter.append("checkout") + previous_nodes = None while True: inprogress = 0 joblist = job_filter.copy() @@ -119,8 +120,13 @@ def watch_jobs(baseurl, token, treeid, job_filter, test): click.secho("No nodes found. Retrying...", fg="yellow") time.sleep(5) continue + if previous_nodes == nodes: + kci_msg_nonl(".") + time.sleep(30) + continue + time_local = time.localtime() - click.echo(f"Current time: {time.strftime('%Y-%m-%d %H:%M:%S', time_local)}") + click.echo(f"\nCurrent time: {time.strftime('%Y-%m-%d %H:%M:%S', time_local)}") click.secho( f"Total tree nodes {len(nodes)} found. job_filter: {job_filter}", fg="green" ) @@ -175,7 +181,8 @@ def watch_jobs(baseurl, token, treeid, job_filter, test): kci_err(f"Test {test} failed: {test_result}") sys.exit(1) - click.echo(f"\rRefresh in 30s...", nl=False) + kci_msg_nonl(f"\rRefresh every 30s...") + previous_nodes = nodes time.sleep(30) From 172c7f7adb54a2974678d88b0a413f3c5adb0173 Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Fri, 17 Jan 2025 08:32:25 -0300 Subject: [PATCH 3/7] bisect: remove api_connection related code The bisect command doesn't call into the Maestro API, so we don't need to manipulate such code here. Signed-off-by: Gustavo Padovan --- kcidev/subcommands/bisect.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/kcidev/subcommands/bisect.py b/kcidev/subcommands/bisect.py index 784cad6..3665cc9 100644 --- a/kcidev/subcommands/bisect.py +++ b/kcidev/subcommands/bisect.py @@ -40,11 +40,6 @@ } -def api_connection(host): - click.secho("api connect: " + host, fg="green") - return host - - def load_state(file="state.json"): if os.path.exists(file): with open(file, "r") as f: @@ -239,7 +234,6 @@ def bisect( ): config = ctx.obj.get("CFG") instance = ctx.obj.get("INSTANCE") - p_url = api_connection(config[instance]["pipeline"]) state_file = os.path.join(workdir, statefile) state = load_state(state_file) From e46e310169e9c7eacb37f6ba3e1228ad324c4cfc Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Thu, 16 Jan 2025 11:35:57 -0300 Subject: [PATCH 4/7] subcommands: create maestro_common files We are already duplicating a lot of code, even in such an earlier phase of the project. Let's change this trend and start consolidating our code base. Signed-off-by: Gustavo Padovan --- kcidev/libs/common.py | 4 ++++ kcidev/libs/maestro_common.py | 14 ++++++++++++++ kcidev/subcommands/checkout.py | 10 +++------- kcidev/subcommands/maestro_results.py | 12 ++++-------- kcidev/subcommands/patch.py | 8 +++----- kcidev/subcommands/testretry.py | 9 +++------ 6 files changed, 31 insertions(+), 26 deletions(-) create mode 100644 kcidev/libs/maestro_common.py diff --git a/kcidev/libs/common.py b/kcidev/libs/common.py index 2fa1fde..d5b62cb 100644 --- a/kcidev/libs/common.py +++ b/kcidev/libs/common.py @@ -86,6 +86,10 @@ def kci_msg(content): click.echo(content) +def kci_log(content): + click.secho(content, err=True) + + def kci_err(content): click.secho(content, fg="red", err=True) diff --git a/kcidev/libs/maestro_common.py b/kcidev/libs/maestro_common.py new file mode 100644 index 0000000..1ab8a04 --- /dev/null +++ b/kcidev/libs/maestro_common.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import json + +import click + +from kcidev.libs.common import * + + +def maestro_print_api_call(host, data=None): + click.secho("maestro api endpoint: " + host, fg="green") + if data: + kci_log(json.dumps(data, indent=4)) diff --git a/kcidev/subcommands/checkout.py b/kcidev/subcommands/checkout.py index efadd87..e2ec001 100644 --- a/kcidev/subcommands/checkout.py +++ b/kcidev/subcommands/checkout.py @@ -13,11 +13,7 @@ from git import Repo from kcidev.libs.common import * - - -def api_connection(host): - click.secho("api connect: " + host, fg="green") - return host +from kcidev.libs.maestro_common import * def display_api_error(response): @@ -46,7 +42,7 @@ def send_checkout_full(baseurl, token, **kwargs): if "platform_filter" in kwargs: data["platformfilter"] = kwargs["platform_filter"] jdata = json.dumps(data) - print(jdata) + maestro_print_api_call(url, data) try: response = requests.post(url, headers=headers, data=jdata, timeout=30) except requests.exceptions.RequestException as e: @@ -247,7 +243,7 @@ def checkout( ): cfg = ctx.obj.get("CFG") instance = ctx.obj.get("INSTANCE") - url = api_connection(cfg[instance]["pipeline"]) + url = cfg[instance]["pipeline"] apiurl = cfg[instance]["api"] token = cfg[instance]["token"] if not job_filter: diff --git a/kcidev/subcommands/maestro_results.py b/kcidev/subcommands/maestro_results.py index 95ffde8..eafc526 100644 --- a/kcidev/subcommands/maestro_results.py +++ b/kcidev/subcommands/maestro_results.py @@ -10,11 +10,7 @@ from git import Repo from kcidev.libs.common import * - - -def api_connection(host): - click.secho("api connect: " + host, fg="green") - return host +from kcidev.libs.maestro_common import * def print_nodes(nodes, field): @@ -35,7 +31,7 @@ def get_node(url, nodeid, field): "Content-Type": "application/json; charset=utf-8", } url = url + "latest/node/" + nodeid - click.secho(url) + maestro_print_api_call(url) response = requests.get(url, headers=headers) try: response.raise_for_status() @@ -59,7 +55,7 @@ def get_nodes(url, limit, offset, filter, field): # if we need anything more complex than eq(=) url = url + "&" + f - click.secho(url) + maestro_print_api_call(url) response = requests.get(url, headers=headers) try: response.raise_for_status() @@ -114,7 +110,7 @@ def get_nodes(url, limit, offset, filter, field): def maestro_results(ctx, nodeid, nodes, limit, offset, filter, field): config = ctx.obj.get("CFG") instance = ctx.obj.get("INSTANCE") - url = api_connection(config[instance]["api"]) + url = config[instance]["api"] if nodeid: get_node(url, nodeid, field) if nodes: diff --git a/kcidev/subcommands/patch.py b/kcidev/subcommands/patch.py index 1ba1c02..b61a5b8 100644 --- a/kcidev/subcommands/patch.py +++ b/kcidev/subcommands/patch.py @@ -8,10 +8,7 @@ import toml from git import Repo - -def api_connection(host): - click.secho("api connect: " + host, fg="green") - return host +from kcidev.libs.maestro_common import * def send_build(url, patch, branch, treeurl, token): @@ -26,6 +23,7 @@ def send_build(url, patch, branch, treeurl, token): "kbuildname": "example", "testname": "example", } + maestro_print_api_call(url, values) response = requests.post(url, headers=headers, files={"patch": patch}, data=values) click.secho(response.status_code, fg="green") click.secho(response.json(), fg="green") @@ -49,7 +47,7 @@ def send_build(url, patch, branch, treeurl, token): def patch(ctx, repository, branch, private, patch): config = ctx.obj.get("CFG") instance = ctx.obj.get("INSTANCE") - url = api_connection(config[instance]["pipeline"]) + url = config[instance]["pipeline"] patch = open(patch, "rb") send_build(url, patch, branch, repository, config[instance]["token"]) diff --git a/kcidev/subcommands/testretry.py b/kcidev/subcommands/testretry.py index 79e01c5..a908913 100644 --- a/kcidev/subcommands/testretry.py +++ b/kcidev/subcommands/testretry.py @@ -8,11 +8,7 @@ from git import Repo from kcidev.libs.common import * - - -def api_connection(host): - click.secho("api connect: " + host, fg="green") - return host +from kcidev.libs.maestro_common import * def display_api_error(response): @@ -32,6 +28,7 @@ def send_jobretry(baseurl, jobid, token): } data = {"nodeid": jobid} jdata = json.dumps(data) + maestro_print_api_call(url, data) try: response = requests.post(url, headers=headers, data=jdata) except requests.exceptions.RequestException as e: @@ -54,7 +51,7 @@ def send_jobretry(baseurl, jobid, token): def testretry(ctx, nodeid): cfg = ctx.obj.get("CFG") instance = ctx.obj.get("INSTANCE") - url = api_connection(cfg[instance]["pipeline"]) + url = cfg[instance]["pipeline"] resp = send_jobretry(url, nodeid, cfg[instance]["token"]) if resp and "message" in resp: click.secho(resp["message"], fg="green") From 4e217f2d08dc22174b6ad4b2030c362cad6cffd6 Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Thu, 16 Jan 2025 11:48:41 -0300 Subject: [PATCH 5/7] maestro_common: move function to show api error in Consolidate our API error message func. This will facilitate future usage. Signed-off-by: Gustavo Padovan --- kcidev/libs/common.py | 4 ++++ kcidev/libs/maestro_common.py | 11 +++++++++++ kcidev/subcommands/checkout.py | 15 ++------------- kcidev/subcommands/testretry.py | 11 +---------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/kcidev/libs/common.py b/kcidev/libs/common.py index d5b62cb..68b3692 100644 --- a/kcidev/libs/common.py +++ b/kcidev/libs/common.py @@ -90,6 +90,10 @@ def kci_log(content): click.secho(content, err=True) +def kci_warning(content): + click.secho(content, fg="yellow", err=True) + + def kci_err(content): click.secho(content, fg="red", err=True) diff --git a/kcidev/libs/maestro_common.py b/kcidev/libs/maestro_common.py index 1ab8a04..fdbd977 100644 --- a/kcidev/libs/maestro_common.py +++ b/kcidev/libs/maestro_common.py @@ -12,3 +12,14 @@ def maestro_print_api_call(host, data=None): click.secho("maestro api endpoint: " + host, fg="green") if data: kci_log(json.dumps(data, indent=4)) + + +def maestro_api_error(response): + kci_err(f"API response error code: {response.status_code}") + try: + kci_err(response.json()) + except json.decoder.JSONDecodeError: + kci_warning(f"No JSON response. Plain text: {response.text}") + except Exception as e: + kci_err(f"API response error: {e}: {response.text}") + return diff --git a/kcidev/subcommands/checkout.py b/kcidev/subcommands/checkout.py index e2ec001..1e0f76f 100644 --- a/kcidev/subcommands/checkout.py +++ b/kcidev/subcommands/checkout.py @@ -16,17 +16,6 @@ from kcidev.libs.maestro_common import * -def display_api_error(response): - kci_err(f"API response error code: {response.status_code}") - try: - kci_err(response.json()) - except json.decoder.JSONDecodeError: - click.secho(f"No JSON response. Plain text: {response.text}", fg="yellow") - except Exception as e: - kci_err(f"API response error: {e}: {response.text}") - return - - def send_checkout_full(baseurl, token, **kwargs): url = baseurl + "api/checkout" headers = { @@ -50,7 +39,7 @@ def send_checkout_full(baseurl, token, **kwargs): return if response.status_code != 200: - display_api_error(response) + maestro_api_error(response) return None return response.json() @@ -71,7 +60,7 @@ def retrieve_treeid_nodes(baseurl, token, treeid): return None if response.status_code >= 400: - display_api_error(response) + maestro_api_error(response) return None return response.json() diff --git a/kcidev/subcommands/testretry.py b/kcidev/subcommands/testretry.py index a908913..c4b1d3e 100644 --- a/kcidev/subcommands/testretry.py +++ b/kcidev/subcommands/testretry.py @@ -11,15 +11,6 @@ from kcidev.libs.maestro_common import * -def display_api_error(response): - kci_err(f"API response error code: {response.status_code}") - try: - kci_err(response.json()) - except json.decoder.JSONDecodeError: - click.secho(f"No JSON response. Plain text: {response.text}", fg="yellow") - return - - def send_jobretry(baseurl, jobid, token): url = baseurl + "api/jobretry" headers = { @@ -36,7 +27,7 @@ def send_jobretry(baseurl, jobid, token): return if response.status_code != 200: - display_api_error(response) + maestro_api_error(response) return None return response.json() From f5613a316db923dcbdf4db5142a321a0f8325937 Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Thu, 16 Jan 2025 13:12:37 -0300 Subject: [PATCH 6/7] maestro_results: return parsable json By returning a json, we can immediatelly parse it in the cmdline with 'jq'. That will help our scripting. Ideally we would have a --json for this, but we can change this later as kci-dev didn't reach 1.0 yet. Suggested-by:Manu Bretelle Signed-off-by: Gustavo Padovan --- kcidev/subcommands/maestro_results.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kcidev/subcommands/maestro_results.py b/kcidev/subcommands/maestro_results.py index eafc526..4fe4578 100644 --- a/kcidev/subcommands/maestro_results.py +++ b/kcidev/subcommands/maestro_results.py @@ -14,6 +14,7 @@ def print_nodes(nodes, field): + res = [] if not isinstance(nodes, list): nodes = [nodes] for node in nodes: @@ -21,9 +22,10 @@ def print_nodes(nodes, field): data = {} for f in field: data[f] = node.get(f) - click.secho(pprint.pprint(data), fg="green", nl=False) + res.append(data) else: - click.secho(pprint.pprint(node), fg="green", nl=False) + res.append(node) + kci_msg(json.dumps(res, sort_keys=True, indent=4)) def get_node(url, nodeid, field): @@ -49,13 +51,13 @@ def get_nodes(url, limit, offset, filter, field): "Content-Type": "application/json; charset=utf-8", } url = url + "latest/nodes/fast?limit=" + str(limit) + "&offset=" + str(offset) + maestro_print_api_call(url) if filter: for f in filter: # TBD: We need to add translate filter to API # if we need anything more complex than eq(=) url = url + "&" + f - maestro_print_api_call(url) response = requests.get(url, headers=headers) try: response.raise_for_status() From 9c13ea6e19f06d6006f81575ffc8a0cca59f451c Mon Sep 17 00:00:00 2001 From: Gustavo Padovan Date: Fri, 17 Jan 2025 08:44:24 -0300 Subject: [PATCH 7/7] checkout: add shorter '-w' option for watch Signed-off-by: Gustavo Padovan --- docs/checkout.md | 2 +- kcidev/subcommands/checkout.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/checkout.md b/docs/checkout.md index 64208d8..26056bd 100644 --- a/docs/checkout.md +++ b/docs/checkout.md @@ -105,7 +105,7 @@ Other options: You can also set instead of --commit option --tipoftree which will retrieve the latest commit of the tree. -### --watch +### --watch, -w Additionally, you can use --watch option to watch the progress of the test. diff --git a/kcidev/subcommands/checkout.py b/kcidev/subcommands/checkout.py index 1e0f76f..bf0be6b 100644 --- a/kcidev/subcommands/checkout.py +++ b/kcidev/subcommands/checkout.py @@ -208,6 +208,7 @@ def retrieve_tot_commit(repourl, branch): ) @click.option( "--watch", + "-w", help="Interactively watch for a tasks in job-filter", is_flag=True, )