squad-compare-builds

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim: set ts=4
#
# Copyright 2022-present Linaro Limited
#
# SPDX-License-Identifier: MIT


import argparse
import fnmatch
import logging
import os
import sys
from urllib import request

import yaml
from squad_client.core.api import SquadApi
from squad_client.core.models import Squad
from squad_client.shortcuts import download_tests as download
from squad_client.shortcuts import get_build

squad_host_url = "https://qa-reports.linaro.org/"
SquadApi.configure(cache=3600, url=os.getenv("SQUAD_HOST", squad_host_url))

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()


def parse_args():
    parser = argparse.ArgumentParser(description="Compare builds within SQUAD")

    parser.add_argument(
        "--gpb",
        required=True,
        action="append",
        nargs=3,
        help="squad group,project,build",
    )

    parser.add_argument(
        "--environments", help="Filter on environments (separated by ',')"
    )

    parser.add_argument("--suites", help="Filter on suites (separated by ',')")

    parser.add_argument(
        "--filename", help="Name of the output file where results will be written"
    )

    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help="Display debug messages",
    )

    return parser.parse_args()


def download_tests(project, build, environments, suites, output_filename):
    download(
        project,
        build,
        environments,
        suites,
        "{test.environment.slug}/{test.test_run.metadata.build_name}/{test.name} {test.status}",
        output_filename,
    )


def run():
    args = parse_args()
    if args.debug:
        logger.setLevel(level=logging.DEBUG)

    files = []
    index = 0
    tests = {}

    known_issue_files = "kselftests-production.yaml kvm-unit-tests.yaml libhugetlbfs-production.yaml ltp-production.yaml network-basic-tests.yaml packetdrill-tests.yaml perf.yaml spectre-meltdown-checker.yaml v4l2-compliance.yaml"
    known_issues_patterns = []
    group_name, project_name, build_name = args.gpb[0]
    for file in known_issue_files.split():

        x = request.urlopen(
            f"https://raw.githubusercontent.com/Linaro/qa-reports-known-issues/master/{file}"
        )
        issues = yaml.safe_load(x)
        for tmp in issues["projects"][0]["known_issues"]:
            environments = []
            projects = []
            for key, value in tmp.items():
                if "environments" in tmp.keys():
                    environments = tmp["environments"]
                else:
                    for matrix in tmp["matrix_apply"]:
                        environments.append(matrix["environments"])

                if "projects" in tmp.keys():
                    projects = tmp["projects"]
                else:
                    for matrix in tmp["matrix_apply"]:
                        projects.append(matrix["projects"])
                if "test_names" in key:
                    for argh in value:
                        if f"{group_name}/{project_name}" in projects:
                            for env in environments:
                                known_issues_patterns.append(
                                    f"{env}/*{argh.replace('.', '?')}"
                                )
                elif "test_name" in key:
                    if f"{group_name}/{project_name}" in projects:
                        for env in environments:
                            known_issues_patterns.append(
                                f"{env}/*{value.replace('.', '?')}"
                            )

    for group_name, project_name, build_name in args.gpb:
        group = Squad().group(group_name)
        project = group.project(project_name)
        build = get_build(build_name, project)

        environments = None
        if args.environments:
            environments = [
                project.environment(e) for e in args.environments.split(",")
            ]

        suites = None
        if args.suites:
            suites = []
            for s in args.suites.split(","):
                suites += project.suites(slug__startswith=s).values()

        file = f"{group.slug}-{project.slug}-{build.version}".replace("~", "")
        file_with_ending = os.path.join(file + ".txt")
        download_tests(project, build, environments, suites, file_with_ending)
        file_open = open(file_with_ending, "r")
        file_lines = file_open.readlines()

        for line in file_lines:
            test_name, test_result = line.split()
            if test_name not in tests.keys():
                tests[test_name] = [None for e in args.gpb]

            tests[test_name][index] = test_result

        index += 1
        files.append(file)

        logger.debug(f"group: {group}, project: {project}, build: {build}")

    table_str = ""
    lines = list()
    for test_name in tests.keys():
        test_results = tests[test_name]
        line = ""
        result = test_results[0]
        use_line = False
        for test in test_results:
            if result != test:
                use_line = True
            line += f"{test} | "

        match = False
        for known_issue in known_issues_patterns:
            if fnmatch.fnmatch(test_name, known_issue):
                match = True
                break
        if match:
            line += "X | "
        else:
            line += "  | "

        line += f"{test_name}"
        if use_line:
            lines.append(line)
    table_str = "\n    ".join(lines)

    table_filename = "table-"
    report = ""
    first = True
    headings = ""
    for file in files:
        if first:
            table_filename += f"{file}"
            first = False
            report = f"Base file: {file}\n"
            headings = "base file"
        else:
            table_filename += f"-vs-{file}"
            report += f"file {files.index(file)}: {file}\n"
            headings += f" | file {files.index(file)}"
    report += f"\n{headings} | known-issue | test_name\n"
    report += "-------------------------------------------------------\n"

    table_filename += ".txt"
    report += f"""
    {table_str}

    table file: {table_filename}"""

    with open(table_filename, "w") as fp:
        fp.write(report)

    print(report)


if __name__ == "__main__":
    sys.exit(run())