Merging pull request 231

Signed-off-by: Lukáš Doktor <ldoktor@redhat.com>

* github.com:distributed-system-analysis/run-perf:
  contrib: Allow http/https pages only in rpm_links.py
  contrib: Simplify the find_rpms function
  docs: Port intersphinx_mapping to sphinx>=1.0
  contrib: Example script to bisect using bisecter
  contrib: Add more colors to extract_data
  contrib: Include all good/bad results
  Actually implement the --host-rpms and --worker-rpms
  Store full args in Controller
  contrib: Circle through colors
  contrib: Sort the result paths to ensure we're getting stable results
  contrib: Set --average argument type

contrib/bisect.sh

@@ -172,7 +172,7 @@ case $1 in
             fi
             idx=$((idx+1))
         done
-        "$@" --html "${DIFFDIR}/report.html" -- "${DIFFDIR}/good" "${RESULTS[@]}" "${DIFFDIR}/bad"
+        "$@" --html "${DIFFDIR}/report.html" -- "${DIFFDIR}/"good* "${RESULTS[@]}" "${DIFFDIR}/"bad*
         echo "${DIFFDIR}/report.html"
         ;;
     "clean")

contrib/bisecter_bisect.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+function usage {
+    echo "usage: $0 BISECTER_WORKDIR CHECK_SCRIPT"
+    echo
+    echo "Can be used to bisect a regression"
+    echo "BISECTER_WORKDIR - path to a directory with bisecter bisection in progress"
+    echo "CHECK_SCRIPT - script to bisecter_bisect_check.sh like script that executes bisect.sh"
+    exit -1
+}
+
+[ "$#" -lt 2 ] && usage
+
+COMPAREPERF="${COMPAREPERF:-compare-perf}"
+
+BISECTER_WORKDIR=$(realpath "$1")
+CHECK_SCRIPT=$(realpath "$2")
+shift; shift
+RUNPERF_DIR=$(pwd)
+SCRIPT_DIR=$(realpath $(dirname "$0"))
+
+"$SCRIPT_DIR"/bisect.sh clean
+pushd "$BISECTER_WORKDIR"
+
+bisecter args || { echo "Bisecter in '$BISECTER_WORKDIR' not started"; exit -1; }
+
+"$CHECK_SCRIPT" good good
+"$CHECK_SCRIPT" bad bad
+bisecter run "$CHECK_SCRIPT" -- check
+BISECT_LOG=$(bisecter log)
+bisecter reset
+rm "$CHECK_SCRIPT"
+popd
+echo
+echo "---< HTML REPORT >---"
+"$SCRIPT_DIR"/bisect.sh report $COMPAREPERF
+echo
+echo "---< BISECT LOG >---"
+echo "$BISECT_LOG"
+echo

contrib/data_analysis/extract_data.py

@@ -49,8 +49,9 @@
 </div>
 """
 
-COLORS = ["#{0:02x}0000", "#00{0:02x}00", "#0000{0:02x}", "#{0:02x}{0:02x}00",
-          "#{0:02x}00{0:02x}", "#00{0:02x}{0:02x}", "#{0:02x}{0:02x}{0:02x}"]
+COLORS = ["#{0:02x}0000", "#{0:02x}0000", "#{0:02x}0000", "#00{0:02x}00",
+          "#00{0:02x}00", "#00{0:02x}00", "#0000{0:02x}", "#0000{0:02x}",
+          "#0000{0:02x}"]
 
 
 def parse_args():
@@ -67,7 +68,8 @@ def parse_args():
     parser.add_argument("-o", "--output", help="Output filename (%(default)s)",
                         default="output.html")
     parser.add_argument("-a", "--average", help="Specify how many values "
-                        "should we average to smooth the curves", default=0)
+                        "should we average to smooth the curves", default=0,
+                        type=int)
     return parser.parse_args()
 
 
@@ -79,7 +81,7 @@ def process(path_glob, variant, out, average):
     name_expr = ['*' in _ for _ in path_glob.split(os.path.sep)]
     color_idx = 0
     no_labels = 0
-    for path in glob.glob(path_glob):
+    for path in sorted(glob.glob(path_glob)):
         split_path = path.split(os.path.sep)
         name = '/'.join([split_path[i]
                          for i in range(len(name_expr)) if name_expr[i]])
@@ -108,7 +110,7 @@ def process(path_glob, variant, out, average):
             no_labels = max(no_labels, len(sample))
         sys.stdout.write("\n")
         color_idx += 1
-        color_idx %= 7
+        color_idx %= len(COLORS)
     out.write(f"        ],\n        labels: {list(range(no_labels))}\n")
     out.write(CHART2)
 

contrib/rpm_links.py

@@ -0,0 +1,100 @@
+#!/bin/env python3
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+#
+# Copyright: Red Hat Inc. 2023
+# Author: Lukas Doktor <ldoktor@redhat.com>
+
+"""Helper to get links to RPMs from a base url"""
+
+import argparse
+import re
+import sys
+import urllib.request
+
+
+def get_filtered_links(page, link_filter=None, name_filter=None):
+    """
+    Get links from http/https page and filter it according to filters
+
+    :param page: Target url
+    :param link_filter: link url filter
+    :param name_filter: link name filter
+    :return: list of links found on the page
+    """
+    if not page.startswith('http://') and not page.startswith('https://'):
+        return []
+    if link_filter is None:
+        link_filter = '[^"]*'
+    if name_filter is None:
+        name_filter = '[^<]*'
+    regex = f"href=\"({link_filter})\"[^>]*>({name_filter})<"
+    sys.stderr.write(f'Looking for {regex} on {page}\n')
+    with urllib.request.urlopen(page) as req:
+        content = req.read().decode('utf-8')
+    return re.findall(regex, content)
+
+
+def find_rpms(url, pkg_names, pkg_filter, arch):
+    """
+    Parse argument into list of links
+
+    :param url: Query a page for links (koji, python -m http.server, ...):
+    :param pkg_names: Look only for links containing this name(s)
+    :param pkg_filter: Look only for links not containing this name(s)
+    :param arch: Look only for rpms of this and noarch type
+    :return: list of individual links (eg.:
+        ["example.org/foo", "example.org/bar"])
+    """
+
+    link_filter = '[^\"]*'
+    if pkg_filter:
+        link_filter = f"(?!.*(?:{'|'.join(pkg_filter)}))"
+    if pkg_names:
+        link_filter += f"(?:{'|'.join(pkg_names)})"
+    if arch:
+        link_filter += f"[^\"]*(?:noarch|{arch})\\.rpm"
+    else:
+        link_filter += "[^\"]*\\.rpm"
+    # Look for rpm_filter-ed rpms on base page
+    links = get_filtered_links(url, link_filter)
+    if links:
+        return [urllib.parse.urljoin(url, link[0]) for link in links]
+    # Look for rpm_filter-ed rpm in all $arch/ links
+    for link in get_filtered_links(url, name_filter=f"{arch}/?"):
+        links = find_rpms(urllib.parse.urljoin(url, link[0]), pkg_names,
+                          pkg_filter, arch)
+        if links:
+            return links
+    raise RuntimeError(f"Unable to find any {link_filter} links in {url}")
+
+
+def main(cmdline=None):
+    """Cmdline handling wrapper to find_rpms"""
+    parser = argparse.ArgumentParser(prog='rpm-links', description='Detects '
+                                     'links to all matching .rpm files '
+                                     'from the base URL(s)')
+    parser.add_argument('--names', '-n', help='List of pkg names', nargs='*')
+    parser.add_argument('--ignore', '-i', help='List names to be ignored '
+                        'out', nargs='*')
+    parser.add_argument('--arch', '-a', help='Target architecture')
+    parser.add_argument('URLs', help='Base url (ensure proper "/" ending if '
+                        'needed)', nargs='+')
+    args = parser.parse_args(cmdline)
+    links = []
+    for url in args.URLs:
+        links.extend(find_rpms(url, args.names, args.ignore, args.arch))
+    print(' '.join(links))
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

contrib/sample_bisecter_bisect.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Use $1 to set bisecter variant id or '--' to get current bisection id
+#
+# bisecter axis are:
+# 0 - distro
+# 1,2,3 - host rpms
+# 4,5 - guest rpms
+
+[ "$1" != '--' ] && ID="$1" || ID="$(bisecter id)"
+CHECK=$2
+shift; shift
+
+SCRIPT_DIR=$(realpath $(dirname "$0"))
+RPM_LINKS="python $SCRIPT_DIR/rpm_links.py --ignore debug docs --arch x86_64"
+
+HOST="SPECIFY HOST HERE"
+PASS="SPECIFY PASSWORD(s) HERE"
+DISTRO=$(bisecter args -i $ID 0)
+HOST_RPMS=$($RPM_LINKS "$(bisecter args -i $ID 1)" "$(bisecter args -i $ID 2)" "$(bisecter args -i $ID 3)")
+WORKER_RPMS=$($RPM_LINKS "$(bisecter args -i $ID 4)" "$(bisecter args -i $ID 5)")
+
+# Run the run-perf and compare-perf
+"$SCRIPT_DIR/bisect.sh" "$CHECK" "$ID" run-perf -v --host-setup-script-reboot --hosts $HOST --default-password $PASS --distro "$DISTRO" --host-rpms $HOST_RPMS --worker-rpms $WORKER_RPMS --profiles 'Localhost' -- 'fio:{"runtime": "10", "targets": "/fio", "block-sizes": "4", "test-types": "read", "samples": "1", "numjobs": "1", "iodepth": "1", "__NAME__": "fio-rot-1j-1i"}'

docs/source/conf.py

@@ -143,7 +143,7 @@
 
 html_theme_options = {'body_max_width': '90%'}
 
-intersphinx_mapping = {'http://docs.python.org/3': None}  # pylint: disable=C0103
+intersphinx_mapping = {'python': ('http://docs.python.org/3', None)}  # pylint: disable=C0103
 
 autoclass_content = 'both'  # pylint: disable=C0103
 

runperf/__init__.py

@@ -117,10 +117,12 @@ def _parse_args():
                         type=parse_host)
     parser.add_argument("--provisioner", help="Use plugin to provision the "
                         "hosts", type=item_with_params)
-    parser.add_argument("--host-rpm", help="Url/path(s) to rpm packages to be "
-                        "installed on host", nargs="+")
-    parser.add_argument("--guest-rpm", help="Url/path(s) to rpm packages to be"
-                        " installed on guest(s)", nargs="+")
+    parser.add_argument("--host-rpms", help="Url/path(s) to rpm packages or "
+                        "page with links to rpms to be installed on host via "
+                        "'dnf install -y '", nargs="+")
+    parser.add_argument("--worker-rpms", help="Url/path(s) to rpm packages or "
+                        "page with links to rpms to be installed on workers "
+                        "via 'dnf install -y '", nargs="+")
     parser.add_argument("--keep-tmp-files", action="store_true", help="Keep "
                         "the temporary files (local/remote)")
     parser.add_argument("--output",

runperf/machine.py

@@ -261,13 +261,8 @@ class Controller:
 
     def __init__(self, args, log):
         self.log = log
-        self._output_dir = args.output  # place to store results
+        self._args = args
         self._provisioner = args.provisioner
-        # path to setup script to be executed per each host
-        self._host_setup_script = args.host_setup_script
-        # path to setup script to be applied on workers
-        self._worker_setup_script = args.worker_setup_script
-        self._host_setup_script_reboot = args.host_setup_script_reboot
         self.default_passwords = args.default_passwords
         self.paths = args.paths
 
@@ -366,11 +361,15 @@ def setup(self):
         self.for_each_host_retry(2, self.hosts, "setup")
 
         # Allow to customize host
-        if self._host_setup_script:
-            with open(self._host_setup_script, encoding="utf-8") as script:
+        if self._args.host_setup_script:
+            with open(self._args.host_setup_script,
+                      encoding="utf-8") as script:
                 self.for_each_host(self.hosts, 'run_script', [script.read()])
-            if self._host_setup_script_reboot:
-                self.for_each_host(self.hosts, "reboot")
+        if self._args.host_rpms:
+            cmd = utils.shell_dnf_install_cmd(self._args.host_rpms)
+            self.for_each_host(self.hosts, 'run_script', [cmd])
+        if self._args.host_setup_script_reboot:
+            self.for_each_host(self.hosts, "reboot")
         shared_pub_key = self.main_host.generate_ssh_key()
         world_versions = []
         for host in self.hosts:
@@ -380,7 +379,7 @@ def setup(self):
 
     def write_metadata(self, key, value):
         """Append the key:value to the RUNPERF_METADATA file"""
-        with open(os.path.join(self._output_dir, "RUNPERF_METADATA"),
+        with open(os.path.join(self._args.output, "RUNPERF_METADATA"),
                   'a', encoding="utf-8") as out:
             out.write(f"\n{key}:")
             out.write(value)
@@ -400,7 +399,7 @@ def wrapper(*args, **kwargs):
                 try:
                     return func(*args, **kwargs)
                 except Exception as exc:
-                    err_path = utils.record_failure(self._output_dir, exc)
+                    err_path = utils.record_failure(self._args.output, exc)
                     try:
                         self.fetch_logs(err_path)
                     except Exception:   # pylint: disable=W0703
@@ -414,12 +413,16 @@ def _apply_profile(self, profile, extra):
         # Allow 5 attempts, one to revert previous profile, one to
         # apply and 3 extra in case one boot fails to get resources
         # (eg. hugepages)
-        if self._worker_setup_script:
-            with open(self._worker_setup_script,
+        setup_script = None
+        if self._args.worker_setup_script:
+            with open(self._args.worker_setup_script,
                       encoding="utf-8") as setup_script_fd:
                 setup_script = setup_script_fd.read()
-        else:
-            setup_script = None
+        if self._args.worker_rpms:
+            if not setup_script:
+                setup_script = '#!/bin/bash\n'
+            setup_script += '\n\nInstall rpms specified by --worker-rpms\n'
+            setup_script += utils.shell_dnf_install_cmd(self._args.worker_rpms)
         self.for_each_host_retry(5, self.hosts, 'apply_profile',
                                  (profile, extra, setup_script,
                                   self.paths))
@@ -473,7 +476,7 @@ def run_test(self, test_class, workers, extra):
         :param workers: list of workers to be made available for execution
         """
         test = test_class(self.main_host, workers,
-                          os.path.join(self._output_dir, self.profile),
+                          os.path.join(self._args.output, self.profile),
                           self.metadata, extra.copy())
         name = test.name
         CONTEXT.set(1, test.output, "Running test")

runperf/utils/__init__.py

@@ -505,6 +505,16 @@ def shell_find_command(session, command):
     return ''
 
 
+def shell_dnf_install_cmd(pkgs):
+    """
+    Helper to generate dnf command to install rpms via dnf
+
+    :param pkgs: List of pkg names to be installed
+    :return: Command to install all packages
+    """
+    escaped_pkgs = [pipes.quote(_) for _ in pkgs]
+    return f"dnf install -y --nobest --skip-broken {' '.join(escaped_pkgs)}"
+
 def wait_for_machine_calms_down(session, timeout=600):
     """
     Wait until 1m system load calms below 1.0

selftests/core/test_machine.py

@@ -63,7 +63,8 @@ def __init__(self, output):
                                   worker_setup_script=__file__,
                                   provisioner=None, host_setup_script=None,
                                   host_setup_script_reboot=False, metadata={},
-                                  output=output)
+                                  output=output, host_rpms=[],
+                                  worker_rpms=[])
         super().__init__(args, mock.Mock())
         # Make sure we will not harm localhost
         for host in self.hosts:

selftests/utils/test_utils.py

@@ -129,6 +129,10 @@ def test_shell_write_content_cmd(self):
         self.assertNotEqual(match, None)
         self.assertEqual(match[1], match[2])
 
+    def test_shell_dnf_install_cmd(self):
+        self.assertEqual("dnf install -y --nobest --skip-broken foo 'b a r'",
+                         utils.shell_dnf_install_cmd(["foo", "b a r"]))
+
     def test_entry_points(self):
         class EP:
             def __init__(self, name=None, loaded_name=None):