Add signposts and profiling support to JetStream2

https://bugs.webkit.org/show_bug.cgi?id=247907
rdar://102328951

Reviewed by Dewei Zhu.

Two things are needed to add profiling support to JetStream:
  - Support for subtests, since by default trying to profile JetStream may sometimes result in the profile being cut off before the test finishes due to file size issues.
  - Signposting to help label generated profiles

In this patch:
  - Added two new fields to plan files in order to allow them to support subtests
                - `subtest_url_format`: a string template for formatting subtest URL parameters
                - `subtests`: a dictionary of suites and tests in the following form:
                         {"suite1": ["test1, test2"]}
                  If a test does not have a suite associated with it, the key is "".
  - Added a new argument to run-benchmarks, `--list-subtests`, that prints out available subtests from a specified plan.
        - Modified and added helper functions to BenchmarkRunner to support listing and validating subtests.
  - Added a new argument to run-benchmarks, `--subtests`, that limits a test with subtest support to only run those subtests.
  - Added a new signposting patch to help annotate generated profiles.

* Tools/Scripts/webkitpy/benchmark_runner/benchmark_runner.py:
(BenchmarkRunner.__init__):
(BenchmarkRunner):
(BenchmarkRunner._find_plan_file):
(BenchmarkRunner._load_plan_data):
(BenchmarkRunner.format_subtests):
(BenchmarkRunner.available_subtests):
(BenchmarkRunner.validate_subtests):
* Tools/Scripts/webkitpy/benchmark_runner/data/patches/signposts/JetStream2.patch: Added.
* Tools/Scripts/webkitpy/benchmark_runner/data/plans/jetstream2.1.plan:
* Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py:
(config_argument_parser):
(run_benchmark_plan):
(list_subtests):
(start):
* Tools/Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py:
(WebDriverBenchmarkRunner._construct_subtest_url):
(WebDriverBenchmarkRunner._run_one_test):
* Tools/Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py:
(WebServerBenchmarkRunner.__init__):
(WebServerBenchmarkRunner):

Canonical link: https://commits.webkit.org/257642@main

Tools/Scripts/webkitpy/benchmark_runner/benchmark_runner.py

@@ -2,6 +2,7 @@
 import logging
 import sys
 import os
+from collections import defaultdict, OrderedDict
 
 from webkitpy.benchmark_runner.benchmark_builder import BenchmarkBuilder
 from webkitpy.benchmark_runner.benchmark_results import BenchmarkResults
@@ -22,50 +23,42 @@ def istext(a):
 class BenchmarkRunner(object):
     name = 'benchmark_runner'
 
-    def __init__(self, plan_file, local_copy, count_override, timeout_override, build_dir, output_file, platform, browser, browser_path, scale_unit=True, show_iteration_values=False, device_id=None, diagnose_dir=None, pgo_profile_output_dir=None, profile_output_dir=None):
-        try:
-            plan_file = self._find_plan_file(plan_file)
-            with open(plan_file, 'r') as fp:
-                self._plan_name = os.path.split(os.path.splitext(plan_file)[0])[1]
-                self._plan = json.load(fp)
-                if not 'options' in self._plan:
-                    self._plan['options'] = {}
-                if local_copy:
-                    self._plan['local_copy'] = local_copy
-                if count_override:
-                    self._plan['count'] = count_override
-                if timeout_override:
-                    self._plan['timeout'] = timeout_override
-                self._browser_driver = BrowserDriverFactory.create(platform, browser)
-                self._browser_path = browser_path
-                self._build_dir = os.path.abspath(build_dir) if build_dir else None
-                self._diagnose_dir = os.path.abspath(diagnose_dir) if diagnose_dir else None
-                if self._diagnose_dir:
-                    os.makedirs(self._diagnose_dir, exist_ok=True)
-                    _log.info('Collecting diagnostics to {}'.format(self._diagnose_dir))
-                self._pgo_profile_output_dir = os.path.abspath(pgo_profile_output_dir) if pgo_profile_output_dir else None
-                if self._pgo_profile_output_dir:
-                    os.makedirs(self._pgo_profile_output_dir, exist_ok=True)
-                    _log.info('Collecting PGO profiles to {}'.format(self._pgo_profile_output_dir))
-                self._profile_output_dir = os.path.abspath(profile_output_dir) if profile_output_dir else None
-                if self._profile_output_dir:
-                    os.makedirs(self._profile_output_dir, exist_ok=True)
-                    _log.info('Collecting profiles to {}'.format(self._profile_output_dir))
-                self._output_file = output_file
-                self._scale_unit = scale_unit
-                self._show_iteration_values = show_iteration_values
-                self._config = self._plan.get('config', {})
-                if device_id:
-                    self._config['device_id'] = device_id
-                self._config['enable_signposts'] = True if self._profile_output_dir else False
-        except IOError as error:
-            _log.error('Can not open plan file: {plan_file} - Error {error}'.format(plan_file=plan_file, error=error))
-            raise error
-        except ValueError as error:
-            _log.error('Plan file: {plan_file} may not follow JSON format - Error {error}'.format(plan_file=plan_file, error=error))
-            raise error
+    def __init__(self, plan_file, local_copy, count_override, timeout_override, build_dir, output_file, platform, browser, browser_path, subtests=None, scale_unit=True, show_iteration_values=False, device_id=None, diagnose_dir=None, pgo_profile_output_dir=None, profile_output_dir=None):
+        self._plan_name, self._plan = BenchmarkRunner._load_plan_data(plan_file)
+        if 'options' not in self._plan:
+            self._plan['options'] = {}
+        if local_copy:
+            self._plan['local_copy'] = local_copy
+        if count_override:
+            self._plan['count'] = count_override
+        if timeout_override:
+            self._plan['timeout'] = timeout_override
+        self._subtests = self.validate_subtests(subtests) if subtests else None
+        self._browser_driver = BrowserDriverFactory.create(platform, browser)
+        self._browser_path = browser_path
+        self._build_dir = os.path.abspath(build_dir) if build_dir else None
+        self._diagnose_dir = os.path.abspath(diagnose_dir) if diagnose_dir else None
+        if self._diagnose_dir:
+            os.makedirs(self._diagnose_dir, exist_ok=True)
+            _log.info('Collecting diagnostics to {}'.format(self._diagnose_dir))
+        self._pgo_profile_output_dir = os.path.abspath(pgo_profile_output_dir) if pgo_profile_output_dir else None
+        if self._pgo_profile_output_dir:
+            os.makedirs(self._pgo_profile_output_dir, exist_ok=True)
+            _log.info('Collecting PGO profiles to {}'.format(self._pgo_profile_output_dir))
+        self._profile_output_dir = os.path.abspath(profile_output_dir) if profile_output_dir else None
+        if self._profile_output_dir:
+            os.makedirs(self._profile_output_dir, exist_ok=True)
+            _log.info('Collecting profiles to {}'.format(self._profile_output_dir))
+        self._output_file = output_file
+        self._scale_unit = scale_unit
+        self._show_iteration_values = show_iteration_values
+        self._config = self._plan.get('config', {})
+        if device_id:
+            self._config['device_id'] = device_id
+        self._config['enable_signposts'] = True if self._profile_output_dir else False
 
-    def _find_plan_file(self, plan_file):
+    @staticmethod
+    def _find_plan_file(plan_file):
         if not os.path.exists(plan_file):
             abs_path = os.path.join(BenchmarkRunner.plan_directory(), plan_file)
             if os.path.exists(abs_path):
@@ -76,6 +69,21 @@ def _find_plan_file(self, plan_file):
                 return abs_path
         return plan_file
 
+    @staticmethod
+    def _load_plan_data(plan_file):
+        plan_file = BenchmarkRunner._find_plan_file(plan_file)
+        try:
+            with open(plan_file, 'r') as fp:
+                plan_name = os.path.split(os.path.splitext(plan_file)[0])[1]
+                plan = json.load(fp)
+                return plan_name, plan
+        except IOError as error:
+            _log.error('Can not open plan file: {plan_file} - Error {error}'.format(plan_file=plan_file, error=error))
+            raise error
+        except ValueError as error:
+            _log.error('Plan file: {plan_file} may not follow JSON format - Error {error}'.format(plan_file=plan_file, error=error))
+            raise error
+
     @staticmethod
     def plan_directory():
         return os.path.join(os.path.dirname(__file__), 'data/plans')
@@ -85,6 +93,52 @@ def available_plans():
         plans = [os.path.splitext(plan_file)[0] for plan_file in os.listdir(BenchmarkRunner.plan_directory()) if plan_file.endswith(".plan")]
         return plans
 
+    @staticmethod
+    def format_subtests(subtests):
+        # An ordered dictionary helps us prioritize listing subtests without a suite (denoted with key: "")
+        subtests = OrderedDict(subtests)
+        subtest_list = []
+        for suite, tests in subtests.items():
+            subtest_list += ['{suite}/{test}'.format(suite=suite, test=test) if suite else test for test in tests]
+        return subtest_list
+
+    @staticmethod
+    def available_subtests(plan_file):
+        plan = BenchmarkRunner._load_plan_data(plan_file)[1]
+        return plan.get('subtests', {})
+
+    def validate_subtests(self, subtests):
+        if 'subtests' not in self._plan:
+            raise Exception('Subtests are not available for the specified plan.')
+        valid_subtests = OrderedDict(self._plan['subtests'])
+        subtests_to_run = defaultdict(list)
+        for subtest in subtests:
+            if '/' in subtest:
+                subtest_suite, subtest_name = subtest.split('/')
+            else:
+                subtest_suite, subtest_name = None, subtest
+            did_append_subtest = False
+            if subtest_suite:
+                if subtest_suite not in valid_subtests:
+                    _log.warning('{} does not belong to a valid suite, skipping'.format(subtest))
+                    continue
+                if subtest_name in valid_subtests[subtest_suite]:
+                    subtests_to_run[subtest_suite].append(subtest_name)
+                    did_append_subtest = True
+            else:  # If no suite is provided, we should do our best to infer one.
+                for suite, tests in valid_subtests.items():
+                    if subtest_name in tests:
+                        subtests_to_run[suite].append(subtest_name)
+                        did_append_subtest = True
+                        break
+            if not did_append_subtest:
+                _log.warning('{} is not a valid subtest, skipping'.format(subtest))
+        if subtests_to_run:
+            _log.info('Running subtests: {}'.format(BenchmarkRunner.format_subtests(subtests_to_run)))
+            return subtests_to_run
+        else:
+            raise Exception('No valid subtests were specified')
+
     def _run_one_test(self, web_root, test_file, iteration):
         raise NotImplementedError('BenchmarkRunner is an abstract class and shouldn\'t be instantiated.')
 

Tools/Scripts/webkitpy/benchmark_runner/data/patches/signposts/JetStream2.patch

@@ -0,0 +1,22 @@
+diff --git a/PerformanceTests/JetStream2/JetStreamDriver.js b/PerformanceTests/JetStream2/JetStreamDriver.js
+index 0bc36c9e62a1..ec4aaa0c2f1d 100644
+--- a/JetStreamDriver.js
++++ b/JetStreamDriver.js
+@@ -645,6 +645,8 @@ class Benchmark {
+         if (RAMification)
+             resetMemoryPeak();
+
++        __signpostStart(this.name)
++
+         let magicFrame;
+         try {
+             magicFrame = JetStream.runCode(code);
+@@ -654,6 +656,8 @@ class Benchmark {
+         }
+         let results = await promise;
+
++        __signpostStop(this.name)
++
+         this.endTime = new Date();
+
+         if (RAMification) {

Tools/Scripts/webkitpy/benchmark_runner/data/plans/jetstream2.1.plan

@@ -1,8 +1,9 @@
 {
     "timeout": 1200,
     "count": 5,
-    "local_git_archive": "PerformanceTests/JetStream2@05ce373a859fbb562ab903f1c706e0843a248169",
-    "github_source": "https://github.com/WebKit/WebKit/tree/05ce373a859fbb562ab903f1c706e0843a248169/PerformanceTests/JetStream2",
+    "local_git_archive": "PerformanceTests/JetStream2@bb52774d658ff6d0f32d1630d9f1682ea1d7457a",
+    "github_source": "https://github.com/WebKit/WebKit/tree/bb52774d658ff6d0f32d1630d9f1682ea1d7457a/PerformanceTests/JetStream2",
+    "signpost_patch": "data/patches/signposts/JetStream2.patch",
     "entry_point": "index.html?report=true",
     "output_file": "jetstream2.result",
     "github_subtree": {
@@ -516,5 +517,78 @@
             {"path": "worker/bomb.js", "mode": "100644", "type": "blob"},
             {"path": "worker/segmentation.js", "mode": "100644", "type": "blob"}
         ]
+    },
+    "subtest_url_format": "test=${TEST}",
+    "subtests": {
+        "": [
+            "Air",
+            "Basic",
+            "ML",
+            "Babylon",
+            "cdjs",
+            "first-inspector-code-load",
+            "multi-inspector-code-load",
+            "Box2D",
+            "octane-code-load",
+            "crypto",
+            "delta-blue",
+            "earley-boyer",
+            "gbemu",
+            "mandreel",
+            "navier-stokes",
+            "pdfjs",
+            "raytrace",
+            "regexp",
+            "richards",
+            "splay",
+            "typescript",
+            "octane-zlib",
+            "FlightPlanner",
+            "OfflineAssembler",
+            "UniPoker",
+            "async-fs",
+            "float-mm.c",
+            "hash-map",
+            "ai-astar",
+            "gaussian-blur",
+            "stanford-crypto-aes",
+            "stanford-crypto-pbkdf2",
+            "stanford-crypto-sha256",
+            "json-stringify-inspector",
+            "json-parse-inspector",
+            "HashSet-wasm",
+            "tsf-wasm",
+            "quicksort-wasm",
+            "gcc-loops-wasm",
+            "richards-wasm",
+            "bomb-workers",
+            "segmentation",
+            "WSL",
+            "hello_world-LJF",
+            "list_search-LJF",
+            "lists-LJF",
+            "string_lists-LJF",
+            "3d-cube-SP",
+            "3d-raytrace-SP",
+            "base64-SP",
+            "crypto-aes-SP",
+            "crypto-md5-SP",
+            "crypto-sha1-SP",
+            "date-format-tofte-SP",
+            "date-format-xparb-SP",
+            "n-body-SP",
+            "regex-dna-SP",
+            "string-unpack-code-SP",
+            "tagcloud-SP",
+            "acorn-wtb",
+            "babylon-wtb",
+            "chai-wtb",
+            "coffeescript-wtb",
+            "espree-wtb",
+            "jshint-wtb",
+            "lebab-wtb",
+            "prepack-wtb",
+            "uglify-js-wtb"
+        ]
     }
-}
+}

Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py

@@ -38,7 +38,7 @@ def default_diagnose_dir():
 
 def config_argument_parser():
     diagnose_directory = default_diagnose_dir()
-    parser = argparse.ArgumentParser(description='Run browser based performance benchmarks. To run a single benchmark in the recommended way, use run-benchmark --plan. To see the vailable benchmarks, use run-benchmark --list-plans. This script passes through the __XPC variables in its environment to the Safari process.')
+    parser = argparse.ArgumentParser(description='Run browser based performance benchmarks. To run a single benchmark in the recommended way, use run-benchmark --plan. To see the available benchmarks, use run-benchmark --list-plans. This script passes through the __XPC variables in its environment to the Safari process.')
     mutual_group = parser.add_mutually_exclusive_group(required=True)
     mutual_group.add_argument('--plan', help='Run a specific benchmark plan (e.g. speedometer, jetstream).')
     mutual_group.add_argument('--list-plans', action='store_true', help='List all available benchmark plans.')
@@ -53,6 +53,8 @@ def config_argument_parser():
     parser.add_argument('--local-copy', help='Path to a local copy of the benchmark (e.g. PerformanceTests/SunSpider/).')
     parser.add_argument('--device-id', default=None, help='Undocumented option for mobile device testing.')
     parser.add_argument('--debug', action='store_true', help='Enable debug logging.')
+    parser.add_argument('--subtests', nargs='*', help='Specify subtests to run (if applicable to the current plan), case-sensitive and separated by spaces. To see available subtests, use --list-subtests with a --plan argument.')
+    parser.add_argument('--list-subtests', action='store_true', help='List valid subtests from the test plan.')
     parser.add_argument('--diagnose-directory', dest='diagnose_dir', default=diagnose_directory, help='Directory for storing diagnose information on test failure. Defaults to {}.'.format(diagnose_directory))
     parser.add_argument('--no-adjust-unit', dest='scale_unit', action='store_false', help="Don't convert to scientific notation.")
     parser.add_argument('--show-iteration-values', dest='show_iteration_values', action='store_true', help="Show the measured value for each iteration in addition to averages.")
@@ -95,7 +97,7 @@ def run_benchmark_plan(args, plan):
     benchmark_runner_class = benchmark_runner_subclasses[args.driver]
     runner = benchmark_runner_class(plan,
                                     args.local_copy, args.count, args.timeout, args.build_dir, args.output_file,
-                                    args.platform, args.browser, args.browser_path, args.scale_unit,
+                                    args.platform, args.browser, args.browser_path, args.subtests, args.scale_unit,
                                     args.show_iteration_values, args.device_id, args.diagnose_dir,
                                     args.diagnose_dir if args.generate_pgo_profiles else None,
                                     args.diagnose_dir if args.profile else None)
@@ -108,6 +110,16 @@ def list_benchmark_plans():
         print("\t%s" % plan)
 
 
+def list_subtests(plan):
+    subtests = BenchmarkRunner.available_subtests(plan)
+    if subtests:
+        print('Available subtests for {}:'.format(plan))
+        for subtest in BenchmarkRunner.format_subtests(subtests):
+            print('\t{}'.format(subtest))
+    else:
+        print('No subtests are available for {}.'.format(plan))
+
+
 def start(args):
     if args.json_file:
         results_json = json.load(open(args.json_file, 'r'))
@@ -143,6 +155,9 @@ def start(args):
     if args.list_plans:
         list_benchmark_plans()
         return
+    if args.list_subtests:
+        list_subtests(args.plan)
+        return
 
     run_benchmark_plan(args, args.plan)
 

Tools/Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py

@@ -1,3 +1,4 @@
+import collections
 import json
 import logging
 
@@ -14,10 +15,20 @@ def _get_result(self, driver):
         result = driver.execute_script("return window.webdriver_results")
         return result
 
+    def _construct_subtest_url(self, subtests):
+        print(subtests)
+        if not subtests or not isinstance(subtests, collections.Mapping) or 'subtest_url_format' not in self._plan:
+            return ''
+        subtest_url = ''
+        for suite, tests in subtests.items():
+            for test in tests:
+                subtest_url = subtest_url + '&' + self._plan['subtest_url_format'].replace('${SUITE}', suite).replace('${TEST}', test)
+        return subtest_url
+
     def _run_one_test(self, web_root, test_file, iteration):
         from selenium.webdriver.support.ui import WebDriverWait
         try:
-            url = 'file://{root}/{plan_name}/{test_file}'.format(root=web_root, plan_name=self._plan_name, test_file=test_file)
+            url = 'file://{root}/{plan_name}/{test_file}{subtest_url}'.format(root=web_root, plan_name=self._plan_name, test_file=test_file, subtest_url=self._construct_subtest_url(self.subtests))
             driver = self._browser_driver.launch_driver(url, self._plan['options'], self._build_dir, self._browser_path)
             _log.info('Waiting on results from web browser')
             result = WebDriverWait(driver, self._plan['timeout'], poll_frequency=1.0).until(self._get_result)