From ba9592be9fb5eb3c9c5c41e6867b1e5a6d693dc8 Mon Sep 17 00:00:00 2001 From: Amara Emerson Date: Mon, 20 Oct 2025 12:58:12 -0700 Subject: [PATCH 1/2] Add interleaved benchmark execution for test-suite This adds support for separating build and test phases, and running benchmarks from multiple builds in an interleaved fashion to control for environmental factors (ambient temperature, general system load etc). New options: * --build-only: Build tests without running them * --test-prebuilt: Run tests from pre-built directory * --build-dir: Specify build directory (used with --test-prebuilt) * --exec-interleaved-builds: Comma-separated list of builds to interleave Usage: 1. Build two compiler versions: lnt runtest test-suite --build-only \ --sandbox /tmp/sandbox-a \ --cc /path/to/clang-a \ --test-suite ~/llvm-test-suite \ ... lnt runtest test-suite --build-only \ --sandbox /tmp/sandbox-b \ --cc /path/to/clang-b \ --test-suite ~/llvm-test-suite \ ... 2. Run with interleaved execution: lnt runtest test-suite \ --sandbox /tmp/results \ --exec-interleaved-builds /tmp/sandbox-a/build,/tmp/sandbox-b/build \ --exec-multisample 3 This runs tests in the pattern: - Sample 0: build-a -> build-b - Sample 1: build-a -> build-b - Sample 2: build-a -> build-b Temporal interleaving controls for environmental changes that could bias results toward one build. Or, test single build: lnt runtest test-suite --test-prebuilt \ --build-dir /tmp/sandbox-a/build \ --exec-multisample 5 Reports are written to each build directory (report.json, test-results.xunit.xml, test-results.csv). --- lnt/tests/test_suite.py | 310 ++++++++++++++++++++++++++++++++++------ 1 file changed, 269 insertions(+), 41 deletions(-) diff --git a/lnt/tests/test_suite.py b/lnt/tests/test_suite.py index e121bc0c..12b264c7 100644 --- a/lnt/tests/test_suite.py +++ b/lnt/tests/test_suite.py @@ -24,6 +24,7 @@ import lnt.testing import lnt.testing.profile import lnt.testing.util.compilers +import lnt.util.ImportData from lnt.testing.util.misc import timestamp from lnt.testing.util.commands import fatal from lnt.testing.util.commands import mkdir_p @@ -186,6 +187,44 @@ def __init__(self): def run_test(self, opts): + # Validate new build/test mode options + if opts.build_only and opts.test_prebuilt: + self._fatal("--build-only and --test-prebuilt are mutually exclusive") + + if opts.test_prebuilt and opts.build_dir is None and not opts.exec_interleaved_builds: + self._fatal("--test-prebuilt requires --build-dir (or use --exec-interleaved-builds)") + + if opts.build_dir and not opts.test_prebuilt and not opts.exec_interleaved_builds: + self._fatal("--build-dir can only be used with --test-prebuilt or --exec-interleaved-builds") + + if opts.exec_interleaved_builds: + # --exec-interleaved-builds implies --test-prebuilt + opts.test_prebuilt = True + # Parse and validate build directories + opts.exec_interleaved_builds_list = [ + os.path.abspath(d.strip()) + for d in opts.exec_interleaved_builds.split(',') + ] + for build_dir in opts.exec_interleaved_builds_list: + if not os.path.exists(build_dir): + self._fatal( + "--exec-interleaved-builds directory does not exist: %r" % + build_dir) + cmakecache = os.path.join(build_dir, 'CMakeCache.txt') + if not os.path.exists(cmakecache): + self._fatal( + "--exec-interleaved-builds directory is not a configured build: %r" % + build_dir) + + if opts.build_dir: + # Validate build directory + opts.build_dir = os.path.abspath(opts.build_dir) + if not os.path.exists(opts.build_dir): + self._fatal("--build-dir does not exist: %r" % opts.build_dir) + cmakecache = os.path.join(opts.build_dir, 'CMakeCache.txt') + if not os.path.exists(cmakecache): + self._fatal("--build-dir is not a configured build: %r" % opts.build_dir) + if opts.cc is not None: opts.cc = resolve_command_path(opts.cc) @@ -207,13 +246,20 @@ def run_test(self, opts): if not os.path.exists(opts.cxx): self._fatal("invalid --cxx argument %r, does not exist" % (opts.cxx)) - - if opts.test_suite_root is None: - self._fatal('--test-suite is required') - if not os.path.exists(opts.test_suite_root): - self._fatal("invalid --test-suite argument, does not exist: %r" % ( - opts.test_suite_root)) - opts.test_suite_root = os.path.abspath(opts.test_suite_root) + else: + # If --cc not specified, CMake will use its default compiler discovery + # We'll validate that a compiler was found after configuration + if opts.cc is None and not opts.test_prebuilt: + logger.info("No --cc specified, will use CMake's default compiler discovery") + + if not opts.test_prebuilt: + # These are only required when building + if opts.test_suite_root is None: + self._fatal('--test-suite is required') + if not os.path.exists(opts.test_suite_root): + self._fatal("invalid --test-suite argument, does not exist: %r" % ( + opts.test_suite_root)) + opts.test_suite_root = os.path.abspath(opts.test_suite_root) if opts.test_suite_externals: if not os.path.exists(opts.test_suite_externals): @@ -241,20 +287,23 @@ def run_test(self, opts): opts.only_test = opts.single_result if opts.only_test: - # --only-test can either point to a particular test or a directory. - # Therefore, test_suite_root + opts.only_test or - # test_suite_root + dirname(opts.only_test) must be a directory. - path = os.path.join(opts.test_suite_root, opts.only_test) - parent_path = os.path.dirname(path) - - if os.path.isdir(path): - opts.only_test = (opts.only_test, None) - elif os.path.isdir(parent_path): - opts.only_test = (os.path.dirname(opts.only_test), - os.path.basename(opts.only_test)) - else: - self._fatal("--only-test argument not understood (must be a " + - " test or directory name)") + if not opts.test_prebuilt: + # Only validate against test_suite_root if we're not in test-prebuilt mode + # --only-test can either point to a particular test or a directory. + # Therefore, test_suite_root + opts.only_test or + # test_suite_root + dirname(opts.only_test) must be a directory. + path = os.path.join(opts.test_suite_root, opts.only_test) + parent_path = os.path.dirname(path) + + if os.path.isdir(path): + opts.only_test = (opts.only_test, None) + elif os.path.isdir(parent_path): + opts.only_test = (os.path.dirname(opts.only_test), + os.path.basename(opts.only_test)) + else: + self._fatal("--only-test argument not understood (must be a " + + " test or directory name)") + # else: in test-prebuilt mode, we'll use only_test as-is for filtering if opts.single_result and not opts.only_test[1]: self._fatal("--single-result must be given a single test name, " @@ -271,25 +320,49 @@ def run_test(self, opts): self.start_time = timestamp() # Work out where to put our build stuff - if opts.timestamp_build: - ts = self.start_time.replace(' ', '_').replace(':', '-') - build_dir_name = "test-%s" % ts + if opts.test_prebuilt and opts.build_dir: + # In test-prebuilt mode with --build-dir, use the specified build directory + basedir = opts.build_dir + elif opts.exec_interleaved_builds: + # For exec-interleaved-builds, each build uses its own directory + # We'll return early from _run_interleaved_builds(), so basedir doesn't matter + basedir = opts.sandbox_path else: - build_dir_name = "build" - basedir = os.path.join(opts.sandbox_path, build_dir_name) + # Normal mode or build-only mode: use sandbox/build or sandbox/test- + if opts.timestamp_build: + ts = self.start_time.replace(' ', '_').replace(':', '-') + build_dir_name = "test-%s" % ts + else: + build_dir_name = "build" + basedir = os.path.join(opts.sandbox_path, build_dir_name) + self._base_path = basedir cmakecache = os.path.join(self._base_path, 'CMakeCache.txt') - self.configured = not opts.run_configure and \ - os.path.exists(cmakecache) + if opts.test_prebuilt: + # In test-prebuilt mode, the build is already configured + self.configured = True + else: + # In normal/build-only mode, check if we should skip reconfiguration + self.configured = not opts.run_configure and \ + os.path.exists(cmakecache) + + # No additional validation needed - CMake will find default compiler if needed + # The validation after _configure_if_needed() will catch if no compiler found # If we are doing diagnostics, skip the usual run and do them now. if opts.diagnose: return self.diagnose() - # configure, so we can extract toolchain information from the cmake - # output. - self._configure_if_needed() + # Handle exec-interleaved-builds mode separately + if opts.exec_interleaved_builds: + return self._run_interleaved_builds(opts) + + # Configure if needed (skip in test-prebuilt mode) + if not opts.test_prebuilt: + # configure, so we can extract toolchain information from the cmake + # output. + self._configure_if_needed() # Verify that we can actually find a compiler before continuing cmake_vars = self._extract_cmake_vars_from_cache() @@ -323,18 +396,37 @@ def run_test(self, opts): fatal("Cannot detect compiler version. Specify --run-order" " to manually define it.") + # Handle --build-only mode + if opts.build_only: + logger.info("Building tests (--build-only mode)...") + self.run(cmake_vars, compile=True, test=False, skip_lit=True) + logger.info("Build complete. Build directory: %s" % self._base_path) + logger.info("Use --test-prebuilt --build-dir %s to run tests." % self._base_path) + return lnt.util.ImportData.no_submit() + # Now do the actual run. reports = [] json_reports = [] - for i in range(max(opts.exec_multisample, opts.compile_multisample)): - c = i < opts.compile_multisample - e = i < opts.exec_multisample - # only gather perf profiles on a single run. - p = i == 0 and opts.use_perf in ('profile', 'all') - run_report, json_data = self.run(cmake_vars, compile=c, test=e, - profile=p) - reports.append(run_report) - json_reports.append(json_data) + # In test-prebuilt mode, we only run tests, no compilation + if opts.test_prebuilt: + for i in range(opts.exec_multisample): + # only gather perf profiles on a single run. + p = i == 0 and opts.use_perf in ('profile', 'all') + run_report, json_data = self.run(cmake_vars, compile=False, test=True, + profile=p) + reports.append(run_report) + json_reports.append(json_data) + else: + # Normal mode: build and test + for i in range(max(opts.exec_multisample, opts.compile_multisample)): + c = i < opts.compile_multisample + e = i < opts.exec_multisample + # only gather perf profiles on a single run. + p = i == 0 and opts.use_perf in ('profile', 'all') + run_report, json_data = self.run(cmake_vars, compile=c, test=e, + profile=p) + reports.append(run_report) + json_reports.append(json_data) report = self._create_merged_report(reports) @@ -362,6 +454,116 @@ def run_test(self, opts): return self.submit(report_path, opts, 'nts') + def _run_interleaved_builds(self, opts): + """Run tests from multiple builds in an interleaved fashion.""" + logger.info("Running interleaved builds mode with %d builds" % + len(opts.exec_interleaved_builds_list)) + + # Collect information about each build + build_infos = [] + for build_dir in opts.exec_interleaved_builds_list: + logger.info("Loading build from: %s" % build_dir) + + # Temporarily set _base_path and configured to this build directory + saved_base_path = self._base_path + saved_configured = self.configured + self._base_path = build_dir + self.configured = True # Build directories are already configured + + # Extract cmake vars from this build + cmake_vars = self._extract_cmake_vars_from_cache() + if "CMAKE_C_COMPILER" not in cmake_vars or \ + not os.path.exists(cmake_vars["CMAKE_C_COMPILER"]): + self._fatal( + "Couldn't find C compiler in build %s (%s)." % + (build_dir, cmake_vars.get("CMAKE_C_COMPILER"))) + + cc_info = self._get_cc_info(cmake_vars) + logger.info(" Compiler: %s %s" % (cc_info['cc_name'], cc_info['cc_build'])) + + build_infos.append({ + 'build_dir': build_dir, + 'cmake_vars': cmake_vars, + 'cc_info': cc_info + }) + + # Restore _base_path and configured + self._base_path = saved_base_path + self.configured = saved_configured + + # Now run tests in interleaved fashion + all_reports = [] + all_json_reports = [] + + for sample_idx in range(opts.exec_multisample): + logger.info("Running sample %d of %d" % (sample_idx + 1, opts.exec_multisample)) + + for build_idx, build_info in enumerate(build_infos): + logger.info(" Testing build %d/%d: %s" % + (build_idx + 1, len(build_infos), build_info['build_dir'])) + + # Set _base_path and configured to this build directory + self._base_path = build_info['build_dir'] + self.configured = True # Build is already configured, skip reconfiguration + + # Run tests (no compilation) + p = sample_idx == 0 and opts.use_perf in ('profile', 'all') + run_report, json_data = self.run( + build_info['cmake_vars'], + compile=False, + test=True, + profile=p + ) + + all_reports.append(run_report) + all_json_reports.append(json_data) + + logger.info("Interleaved testing complete. Generating reports...") + + # For now, we'll create separate reports for each build + # Group reports by build + reports_by_build = {} + json_by_build = {} + for i, (report, json_data) in enumerate(zip(all_reports, all_json_reports)): + build_idx = i % len(build_infos) + if build_idx not in reports_by_build: + reports_by_build[build_idx] = [] + json_by_build[build_idx] = [] + reports_by_build[build_idx].append(report) + json_by_build[build_idx].append(json_data) + + # Write reports for each build to its own directory + for build_idx, build_info in enumerate(build_infos): + build_dir = build_info['build_dir'] + logger.info("Writing report for build: %s" % build_dir) + + # Merge reports for this build + merged_report = self._create_merged_report(reports_by_build[build_idx]) + + # Write JSON report to build directory + report_path = os.path.join(build_dir, 'report.json') + with open(report_path, 'w') as fd: + fd.write(merged_report.render()) + logger.info(" Report: %s" % report_path) + + # Write xUnit XML to build directory + xml_path = os.path.join(build_dir, 'test-results.xunit.xml') + str_template = _lit_json_to_xunit_xml(json_by_build[build_idx]) + with open(xml_path, 'w') as fd: + fd.write(str_template) + + # Write CSV to build directory + csv_path = os.path.join(build_dir, 'test-results.csv') + str_template = _lit_json_to_csv(json_by_build[build_idx]) + with open(csv_path, 'w') as fd: + fd.write(str_template) + + logger.info("Reports written to each build directory.") + logger.info("To submit results, use 'lnt submit' with each report file.") + + # Return no_submit since we have multiple reports + return lnt.util.ImportData.no_submit() + def _configure_if_needed(self): mkdir_p(self._base_path) if not self.configured: @@ -369,7 +571,7 @@ def _configure_if_needed(self): self._clean(self._base_path) self.configured = True - def run(self, cmake_vars, compile=True, test=True, profile=False): + def run(self, cmake_vars, compile=True, test=True, profile=False, skip_lit=False): mkdir_p(self._base_path) # FIXME: should we only run PGO collection once, even when @@ -389,6 +591,9 @@ def run(self, cmake_vars, compile=True, test=True, profile=False): self._install_benchmark(self._base_path) self.compiled = True + if skip_lit: + return None, None + data = self._lit(self._base_path, test, profile) return self._parse_lit_output(self._base_path, data, cmake_vars), data @@ -1148,6 +1353,29 @@ def diagnose(self): is_flag=True, default=False,) @click.option("--remote-host", metavar="HOST", help="Run tests on a remote machine") +@click.option("--build-only", "build_only", + help="Only build the tests, don't run them. Useful for " + "preparing builds for later interleaved execution.", + is_flag=True, default=False) +@click.option("--test-prebuilt", "test_prebuilt", + help="Only run tests from pre-built directory, skip configure " + "and build steps. Use with --build-dir to specify the " + "build directory.", + is_flag=True, default=False) +@click.option("--build-dir", "build_dir", + metavar="PATH", + help="Path to pre-built test directory (used with --test-prebuilt). " + "This is the actual build directory (e.g., sandbox/build), " + "not the sandbox parent directory.", + type=click.UNPROCESSED, default=None) +@click.option("--exec-interleaved-builds", "exec_interleaved_builds", + metavar="BUILD1,BUILD2,...", + help="Comma-separated list of build directories to interleave " + "execution from. Implies --test-prebuilt. Each path should be " + "a build directory (e.g., sandbox/build). For each multisample, " + "runs all tests from each build in sequence to control for " + "environmental changes.", + type=click.UNPROCESSED, default=None) # Output Options @click.option("--auto-name/--no-auto-name", "auto_name", default=True, show_default=True, help="Whether to automatically derive the submission name") From a1fe4623f83a0acad3561b85d986d1add8a437b5 Mon Sep 17 00:00:00 2001 From: Amara Emerson Date: Mon, 20 Oct 2025 22:45:08 -0700 Subject: [PATCH 2/2] Generalize --build-dir to work with all build modes The --build-dir option now works with all build modes, not just --test-prebuilt. It can be used to specify a custom build directory location, overriding the default sandbox/build or timestamped directory. --- lnt/tests/test_suite.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/lnt/tests/test_suite.py b/lnt/tests/test_suite.py index 12b264c7..a34e65e5 100644 --- a/lnt/tests/test_suite.py +++ b/lnt/tests/test_suite.py @@ -194,9 +194,6 @@ def run_test(self, opts): if opts.test_prebuilt and opts.build_dir is None and not opts.exec_interleaved_builds: self._fatal("--test-prebuilt requires --build-dir (or use --exec-interleaved-builds)") - if opts.build_dir and not opts.test_prebuilt and not opts.exec_interleaved_builds: - self._fatal("--build-dir can only be used with --test-prebuilt or --exec-interleaved-builds") - if opts.exec_interleaved_builds: # --exec-interleaved-builds implies --test-prebuilt opts.test_prebuilt = True @@ -217,13 +214,17 @@ def run_test(self, opts): build_dir) if opts.build_dir: - # Validate build directory + # Validate and normalize build directory path opts.build_dir = os.path.abspath(opts.build_dir) - if not os.path.exists(opts.build_dir): - self._fatal("--build-dir does not exist: %r" % opts.build_dir) - cmakecache = os.path.join(opts.build_dir, 'CMakeCache.txt') - if not os.path.exists(cmakecache): - self._fatal("--build-dir is not a configured build: %r" % opts.build_dir) + if opts.test_prebuilt: + # In test-prebuilt mode, build directory must already exist + if not os.path.exists(opts.build_dir): + self._fatal("--build-dir does not exist: %r" % opts.build_dir) + cmakecache = os.path.join(opts.build_dir, 'CMakeCache.txt') + if not os.path.exists(cmakecache): + self._fatal("--build-dir is not a configured build: %r" % opts.build_dir) + # In normal build mode, --build-dir specifies where to create the build + # (directory will be created if it doesn't exist) if opts.cc is not None: opts.cc = resolve_command_path(opts.cc) @@ -320,15 +321,15 @@ def run_test(self, opts): self.start_time = timestamp() # Work out where to put our build stuff - if opts.test_prebuilt and opts.build_dir: - # In test-prebuilt mode with --build-dir, use the specified build directory + if opts.build_dir: + # If --build-dir is specified, use it (for any build type) basedir = opts.build_dir elif opts.exec_interleaved_builds: # For exec-interleaved-builds, each build uses its own directory # We'll return early from _run_interleaved_builds(), so basedir doesn't matter basedir = opts.sandbox_path else: - # Normal mode or build-only mode: use sandbox/build or sandbox/test- + # Normal mode: use sandbox/build or sandbox/test- if opts.timestamp_build: ts = self.start_time.replace(' ', '_').replace(':', '-') build_dir_name = "test-%s" % ts @@ -1364,9 +1365,10 @@ def diagnose(self): is_flag=True, default=False) @click.option("--build-dir", "build_dir", metavar="PATH", - help="Path to pre-built test directory (used with --test-prebuilt). " - "This is the actual build directory (e.g., sandbox/build), " - "not the sandbox parent directory.", + help="Specify build directory location. With --test-prebuilt, this must " + "be an existing configured build. With --build-only or normal mode, " + "specifies where to create the build (overrides default sandbox/build " + "or timestamped directory). Path can be absolute or relative to sandbox.", type=click.UNPROCESSED, default=None) @click.option("--exec-interleaved-builds", "exec_interleaved_builds", metavar="BUILD1,BUILD2,...",