General improvements to maestro commands

docs/checkout.md

@@ -105,7 +105,7 @@ Other options:
 
 You can also set instead of --commit option --tipoftree which will retrieve the latest commit of the tree.
 
-### --watch
+### --watch, -w
 
 Additionally, you can use --watch option to watch the progress of the test.
 

kcidev/libs/common.py

@@ -86,6 +86,14 @@ def kci_msg(content):
     click.echo(content)
 
 
+def kci_log(content):
+    click.secho(content, err=True)
+
+
+def kci_warning(content):
+    click.secho(content, fg="yellow", err=True)
+
+
 def kci_err(content):
     click.secho(content, fg="red", err=True)
 

kcidev/libs/maestro_common.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import json
+
+import click
+
+from kcidev.libs.common import *
+
+
+def maestro_print_api_call(host, data=None):
+    click.secho("maestro api endpoint: " + host, fg="green")
+    if data:
+        kci_log(json.dumps(data, indent=4))
+
+
+def maestro_api_error(response):
+    kci_err(f"API response error code: {response.status_code}")
+    try:
+        kci_err(response.json())
+    except json.decoder.JSONDecodeError:
+        kci_warning(f"No JSON response. Plain text: {response.text}")
+    except Exception as e:
+        kci_err(f"API response error: {e}: {response.text}")
+    return

kcidev/subcommands/bisect.py

@@ -40,11 +40,6 @@
 }
 
 
-def api_connection(host):
-    click.secho("api connect: " + host, fg="green")
-    return host
-
-
 def load_state(file="state.json"):
     if os.path.exists(file):
         with open(file, "r") as f:
@@ -239,7 +234,6 @@ def bisect(
 ):
     config = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    p_url = api_connection(config[instance]["pipeline"])
 
     state_file = os.path.join(workdir, statefile)
     state = load_state(state_file)

kcidev/subcommands/checkout.py

@@ -13,22 +13,7 @@
 from git import Repo
 
 from kcidev.libs.common import *
-
-
-def api_connection(host):
-    click.secho("api connect: " + host, fg="green")
-    return host
-
-
-def display_api_error(response):
-    kci_err(f"API response error code: {response.status_code}")
-    try:
-        kci_err(response.json())
-    except json.decoder.JSONDecodeError:
-        click.secho(f"No JSON response. Plain text: {response.text}", fg="yellow")
-    except Exception as e:
-        kci_err(f"API response error: {e}: {response.text}")
-    return
+from kcidev.libs.maestro_common import *
 
 
 def send_checkout_full(baseurl, token, **kwargs):
@@ -46,15 +31,15 @@ def send_checkout_full(baseurl, token, **kwargs):
     if "platform_filter" in kwargs:
         data["platformfilter"] = kwargs["platform_filter"]
     jdata = json.dumps(data)
-    print(jdata)
+    maestro_print_api_call(url, data)
     try:
         response = requests.post(url, headers=headers, data=jdata, timeout=30)
     except requests.exceptions.RequestException as e:
         kci_err(f"API connection error: {e}")
         return
 
     if response.status_code != 200:
-        display_api_error(response)
+        maestro_api_error(response)
         return None
     return response.json()
 
@@ -75,7 +60,7 @@ def retrieve_treeid_nodes(baseurl, token, treeid):
         return None
 
     if response.status_code >= 400:
-        display_api_error(response)
+        maestro_api_error(response)
         return None
 
     return response.json()
@@ -111,6 +96,7 @@ def watch_jobs(baseurl, token, treeid, job_filter, test):
     # we need to add to job_filter "checkout" node
     job_filter = list(job_filter)
     job_filter.append("checkout")
+    previous_nodes = None
     while True:
         inprogress = 0
         joblist = job_filter.copy()
@@ -119,8 +105,13 @@ def watch_jobs(baseurl, token, treeid, job_filter, test):
             click.secho("No nodes found. Retrying...", fg="yellow")
             time.sleep(5)
             continue
+        if previous_nodes == nodes:
+            kci_msg_nonl(".")
+            time.sleep(30)
+            continue
+
         time_local = time.localtime()
-        click.echo(f"Current time: {time.strftime('%Y-%m-%d %H:%M:%S', time_local)}")
+        click.echo(f"\nCurrent time: {time.strftime('%Y-%m-%d %H:%M:%S', time_local)}")
         click.secho(
             f"Total tree nodes {len(nodes)} found. job_filter: {job_filter}", fg="green"
         )
@@ -152,7 +143,7 @@ def watch_jobs(baseurl, token, treeid, job_filter, test):
                         sys.exit(2)
                 nodeid = node.get("id")
                 click.secho(
-                    f"Node {nodeid} job {node['name']} State {node['state']} Result {node['result']}",
+                    f"Node: {nodeid} job: {node['name']} State: {node['state']} Result: {node['result']}",
                     fg=color,
                 )
         if len(joblist) == 0 and inprogress == 0:
@@ -175,7 +166,8 @@ def watch_jobs(baseurl, token, treeid, job_filter, test):
                     kci_err(f"Test {test} failed: {test_result}")
                     sys.exit(1)
 
-        click.echo(f"\rRefresh in 30s...", nl=False)
+        kci_msg_nonl(f"\rRefresh every 30s...")
+        previous_nodes = nodes
         time.sleep(30)
 
 
@@ -216,6 +208,7 @@ def retrieve_tot_commit(repourl, branch):
 )
 @click.option(
     "--watch",
+    "-w",
     help="Interactively watch for a tasks in job-filter",
     is_flag=True,
 )
@@ -240,7 +233,7 @@ def checkout(
 ):
     cfg = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    url = api_connection(cfg[instance]["pipeline"])
+    url = cfg[instance]["pipeline"]
     apiurl = cfg[instance]["api"]
     token = cfg[instance]["token"]
     if not job_filter:

kcidev/subcommands/maestro_results.py

@@ -10,32 +10,30 @@
 from git import Repo
 
 from kcidev.libs.common import *
-
-
-def api_connection(host):
-    click.secho("api connect: " + host, fg="green")
-    return host
+from kcidev.libs.maestro_common import *
 
 
 def print_nodes(nodes, field):
+    res = []
     if not isinstance(nodes, list):
         nodes = [nodes]
     for node in nodes:
         if field:
             data = {}
             for f in field:
                 data[f] = node.get(f)
-            click.secho(pprint.pprint(data), fg="green", nl=False)
+            res.append(data)
         else:
-            click.secho(pprint.pprint(node), fg="green", nl=False)
+            res.append(node)
+        kci_msg(json.dumps(res, sort_keys=True, indent=4))
 
 
 def get_node(url, nodeid, field):
     headers = {
         "Content-Type": "application/json; charset=utf-8",
     }
     url = url + "latest/node/" + nodeid
-    click.secho(url)
+    maestro_print_api_call(url)
     response = requests.get(url, headers=headers)
     try:
         response.raise_for_status()
@@ -53,13 +51,13 @@ def get_nodes(url, limit, offset, filter, field):
         "Content-Type": "application/json; charset=utf-8",
     }
     url = url + "latest/nodes/fast?limit=" + str(limit) + "&offset=" + str(offset)
+    maestro_print_api_call(url)
     if filter:
         for f in filter:
             # TBD: We need to add translate filter to API
             # if we need anything more complex than eq(=)
             url = url + "&" + f
 
-    click.secho(url)
     response = requests.get(url, headers=headers)
     try:
         response.raise_for_status()
@@ -114,7 +112,7 @@ def get_nodes(url, limit, offset, filter, field):
 def maestro_results(ctx, nodeid, nodes, limit, offset, filter, field):
     config = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    url = api_connection(config[instance]["api"])
+    url = config[instance]["api"]
     if nodeid:
         get_node(url, nodeid, field)
     if nodes:

kcidev/subcommands/patch.py

@@ -8,10 +8,7 @@
 import toml
 from git import Repo
 
-
-def api_connection(host):
-    click.secho("api connect: " + host, fg="green")
-    return host
+from kcidev.libs.maestro_common import *
 
 
 def send_build(url, patch, branch, treeurl, token):
@@ -26,6 +23,7 @@ def send_build(url, patch, branch, treeurl, token):
         "kbuildname": "example",
         "testname": "example",
     }
+    maestro_print_api_call(url, values)
     response = requests.post(url, headers=headers, files={"patch": patch}, data=values)
     click.secho(response.status_code, fg="green")
     click.secho(response.json(), fg="green")
@@ -49,7 +47,7 @@ def send_build(url, patch, branch, treeurl, token):
 def patch(ctx, repository, branch, private, patch):
     config = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    url = api_connection(config[instance]["pipeline"])
+    url = config[instance]["pipeline"]
     patch = open(patch, "rb")
     send_build(url, patch, branch, repository, config[instance]["token"])
 

kcidev/subcommands/testretry.py

@@ -8,20 +8,7 @@
 from git import Repo
 
 from kcidev.libs.common import *
-
-
-def api_connection(host):
-    click.secho("api connect: " + host, fg="green")
-    return host
-
-
-def display_api_error(response):
-    kci_err(f"API response error code: {response.status_code}")
-    try:
-        kci_err(response.json())
-    except json.decoder.JSONDecodeError:
-        click.secho(f"No JSON response. Plain text: {response.text}", fg="yellow")
-    return
+from kcidev.libs.maestro_common import *
 
 
 def send_jobretry(baseurl, jobid, token):
@@ -32,14 +19,15 @@ def send_jobretry(baseurl, jobid, token):
     }
     data = {"nodeid": jobid}
     jdata = json.dumps(data)
+    maestro_print_api_call(url, data)
     try:
         response = requests.post(url, headers=headers, data=jdata)
     except requests.exceptions.RequestException as e:
         kci_err(f"API connection error: {e}")
         return
 
     if response.status_code != 200:
-        display_api_error(response)
+        maestro_api_error(response)
         return None
     return response.json()
 
@@ -54,7 +42,7 @@ def send_jobretry(baseurl, jobid, token):
 def testretry(ctx, nodeid):
     cfg = ctx.obj.get("CFG")
     instance = ctx.obj.get("INSTANCE")
-    url = api_connection(cfg[instance]["pipeline"])
+    url = cfg[instance]["pipeline"]
     resp = send_jobretry(url, nodeid, cfg[instance]["token"])
     if resp and "message" in resp:
         click.secho(resp["message"], fg="green")