diff --git a/.buildkite/pipeline_pr.py b/.buildkite/pipeline_pr.py index 1a73d523a0e..5a4481b267d 100755 --- a/.buildkite/pipeline_pr.py +++ b/.buildkite/pipeline_pr.py @@ -33,7 +33,7 @@ def get_changed_files(branch): "platforms": DEFAULT_PLATFORMS, # buildkite step parameters "priority": DEFAULT_PRIORITY, - "timeout_in_minutes": 30, + "timeout_in_minutes": 45, } build_grp = group( diff --git a/CHANGELOG.md b/CHANGELOG.md index 30de845eeca..3a99485aa67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ - Added a `cpu-template-helper` tool for assisting with creating and managing custom CPU templates. +### Changed + +- Set FDP_EXCPTN_ONLY bit (CPUID.7h.0:EBX[6]) and ZERO_FCS_FDS bit + (CPUID.7h.0:EBX[13]) in Intel's CPUID normalization process. + ### Fixed - Fixed feature flags in T2S CPU template on Intel Ice Lake. @@ -24,6 +29,11 @@ thread due to a misconfiguration of the `api_event_fd`. - Fixed CPUID leaf 0x1 to disable perfmon and debug feature on x86 host. - Fixed passing through cache information from host in CPUID leaf 0x80000006. +- Fixed the T2S CPU template to set the RRSBA bit of the IA32_ARCH_CAPABILITIES + MSR to 1 in accordance with an Intel microcode update. +- Fixed the T2CL CPU template to pass through the RSBA and RRSBA bits of the + IA32_ARCH_CAPABILITIES MSR from the host in accordance with an Intel microcode + update. ## [1.3.0] diff --git a/docs/cpu_templates/cpuid-normalization.md b/docs/cpu_templates/cpuid-normalization.md index 96779454a26..a9ef1ac538c 100644 --- a/docs/cpu_templates/cpuid-normalization.md +++ b/docs/cpu_templates/cpuid-normalization.md @@ -30,10 +30,12 @@ See also: [boot protocol settings](boot-protocol.md) ## Intel-specific CPUID normalization | Description | Leaf | Subleaf | Register | Bits | -|----------------------------------------------------------------|:----------------------------------:|:-------:|:------------------:|:----:| +|----------------------------------------------------------------|:----------------------------------:|:-------:|:------------------:|:-----:| | Update deterministic cache parameters | 0x4 | all | EAX | 31:14 | | Disable Intel Turbo Boost technology | 0x6 | - | EAX | 1 | | Disable frequency selection | 0x6 | - | ECX | 3 | +| Set FDP_EXCPTN_ONLY bit | 0x7 | 0x0 | EBX | 6 | +| Set "Deprecates FPU CS and FPU DS values" bit | 0x7 | 0x0 | EBX | 13 | | Disable performance monitoring | 0xa | - | EAX, EBX, ECX, EDX | all | | Update brand string to use a default format and real frequency | 0x80000002, 0x80000003, 0x80000004 | - | EAX, EBX, ECX, EDX | all | diff --git a/resources/tests/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_4.14host.json b/resources/tests/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_4.14host.json index a6d7816fb42..71deedbf335 100644 --- a/resources/tests/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_4.14host.json +++ b/resources/tests/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_4.14host.json @@ -270,7 +270,7 @@ }, { "register": "ebx", - "bitmap": "0b11010001100111110100011110101011" + "bitmap": "0b11010001100111110110011111101011" }, { "register": "ecx", diff --git a/resources/tests/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_5.10host.json b/resources/tests/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_5.10host.json index ea6cd64066d..55b06696157 100644 --- a/resources/tests/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_5.10host.json +++ b/resources/tests/cpu_template_helper/fingerprint_INTEL_CASCADELAKE_5.10host.json @@ -270,7 +270,7 @@ }, { "register": "ebx", - "bitmap": "0b11010001100111110100011110101011" + "bitmap": "0b11010001100111110110011111101011" }, { "register": "ecx", diff --git a/resources/tests/cpu_template_helper/fingerprint_INTEL_ICELAKE_4.14host.json b/resources/tests/cpu_template_helper/fingerprint_INTEL_ICELAKE_4.14host.json index 33fc6640956..24fc2302afa 100644 --- a/resources/tests/cpu_template_helper/fingerprint_INTEL_ICELAKE_4.14host.json +++ b/resources/tests/cpu_template_helper/fingerprint_INTEL_ICELAKE_4.14host.json @@ -270,7 +270,7 @@ }, { "register": "ebx", - "bitmap": "0b11110001101111110000011110101011" + "bitmap": "0b11110001101111110010011111101011" }, { "register": "ecx", diff --git a/resources/tests/cpu_template_helper/fingerprint_INTEL_ICELAKE_5.10host.json b/resources/tests/cpu_template_helper/fingerprint_INTEL_ICELAKE_5.10host.json index 815b2fef87d..f76a1de5a52 100644 --- a/resources/tests/cpu_template_helper/fingerprint_INTEL_ICELAKE_5.10host.json +++ b/resources/tests/cpu_template_helper/fingerprint_INTEL_ICELAKE_5.10host.json @@ -270,7 +270,7 @@ }, { "register": "ebx", - "bitmap": "0b11110001101111110000011110101011" + "bitmap": "0b11110001101111110010011111101011" }, { "register": "ecx", diff --git a/resources/tests/cpu_template_helper/fingerprint_INTEL_SKYLAKE_4.14host.json b/resources/tests/cpu_template_helper/fingerprint_INTEL_SKYLAKE_4.14host.json index e6911d0ec25..479c61364d8 100644 --- a/resources/tests/cpu_template_helper/fingerprint_INTEL_SKYLAKE_4.14host.json +++ b/resources/tests/cpu_template_helper/fingerprint_INTEL_SKYLAKE_4.14host.json @@ -270,7 +270,7 @@ }, { "register": "ebx", - "bitmap": "0b11010001100111110100111110111011" + "bitmap": "0b11010001100111110110111111111011" }, { "register": "ecx", diff --git a/resources/tests/cpu_template_helper/fingerprint_INTEL_SKYLAKE_5.10host.json b/resources/tests/cpu_template_helper/fingerprint_INTEL_SKYLAKE_5.10host.json index 6a24cb8ba15..1e020eebb45 100644 --- a/resources/tests/cpu_template_helper/fingerprint_INTEL_SKYLAKE_5.10host.json +++ b/resources/tests/cpu_template_helper/fingerprint_INTEL_SKYLAKE_5.10host.json @@ -270,7 +270,7 @@ }, { "register": "ebx", - "bitmap": "0b11010001100111110100111110111011" + "bitmap": "0b11010001100111110110111111111011" }, { "register": "ecx", diff --git a/resources/tests/static_cpu_templates/c3.json b/resources/tests/static_cpu_templates/c3.json index 164815459a8..dcaebe0e5f5 100644 --- a/resources/tests/static_cpu_templates/c3.json +++ b/resources/tests/static_cpu_templates/c3.json @@ -26,7 +26,7 @@ "modifiers": [ { "register": "ebx", - "bitmap": "0b000000000x0x000000x000x0x00000xx" + "bitmap": "0b000000000x0x000000x000x0xx0000xx" }, { "register": "ecx", diff --git a/resources/tests/static_cpu_templates/t2.json b/resources/tests/static_cpu_templates/t2.json index 3d4ef835609..907c76f1274 100644 --- a/resources/tests/static_cpu_templates/t2.json +++ b/resources/tests/static_cpu_templates/t2.json @@ -26,7 +26,7 @@ "modifiers": [ { "register": "ebx", - "bitmap": "0b00000000000x000000000x1xx0x0x0xx" + "bitmap": "0b00000000000x000000x00x1xxxx0x0xx" }, { "register": "ecx", diff --git a/resources/tests/static_cpu_templates/t2a.json b/resources/tests/static_cpu_templates/t2a.json index 8d0eb36f39c..b25b1011378 100644 --- a/resources/tests/static_cpu_templates/t2a.json +++ b/resources/tests/static_cpu_templates/t2a.json @@ -26,7 +26,7 @@ "modifiers": [ { "register": "ebx", - "bitmap": "0b00000000000x000000000x1xx0x0x0xx" + "bitmap": "0b00000000000x000000x00x1xxxx0x0xx" }, { "register": "ecx", diff --git a/resources/tests/static_cpu_templates/t2cl.json b/resources/tests/static_cpu_templates/t2cl.json index fca11e1d360..6236cdb09c6 100644 --- a/resources/tests/static_cpu_templates/t2cl.json +++ b/resources/tests/static_cpu_templates/t2cl.json @@ -26,7 +26,7 @@ "modifiers": [ { "register": "ebx", - "bitmap": "0b00000000000x000000000x1xx0x0x0xx" + "bitmap": "0b00000000000x000000x00x1xxxx0x0xx" }, { "register": "ecx", @@ -90,7 +90,7 @@ "msr_modifiers": [ { "addr": "0x10a", - "bitmap": "0b0000000000000000000000000000000000000000000000000000000011101011" + "bitmap": "0b00000000000000000000000000000000000000000000x0000000000011101x11" } ] } diff --git a/resources/tests/static_cpu_templates/t2s.json b/resources/tests/static_cpu_templates/t2s.json index bb4dd8358a9..d1fee0f0ee8 100644 --- a/resources/tests/static_cpu_templates/t2s.json +++ b/resources/tests/static_cpu_templates/t2s.json @@ -26,7 +26,7 @@ "modifiers": [ { "register": "ebx", - "bitmap": "0b00000000000x000000000x1xx0x0x0xx" + "bitmap": "0b00000000000x000000x00x1xxxx0x0xx" }, { "register": "ecx", @@ -90,7 +90,7 @@ "msr_modifiers": [ { "addr": "0x10a", - "bitmap": "0b0000000000000000000000000000000000000000000000000000110001001100" + "bitmap": "0b0000000000000000000000000000000000000000000010000000110001001100" } ] } diff --git a/src/vmm/src/cpu_config/x86_64/cpuid/intel/normalize.rs b/src/vmm/src/cpu_config/x86_64/cpuid/intel/normalize.rs index 59ae40f59c2..ebee7058cc9 100644 --- a/src/vmm/src/cpu_config/x86_64/cpuid/intel/normalize.rs +++ b/src/vmm/src/cpu_config/x86_64/cpuid/intel/normalize.rs @@ -18,6 +18,9 @@ pub enum NormalizeCpuidError { /// Leaf 0x6 is missing from CPUID. #[error("Leaf 0x6 is missing from CPUID.")] MissingLeaf6, + /// Leaf 0x7 / subleaf 0 is missing from CPUID. + #[error("Leaf 0x7 / subleaf 0 is missing from CPUID.")] + MissingLeaf7, /// Leaf 0xA is missing from CPUID. #[error("Leaf 0xA is missing from CPUID.")] MissingLeafA, @@ -85,6 +88,7 @@ impl super::IntelCpuid { ) -> Result<(), NormalizeCpuidError> { self.update_deterministic_cache_entry(cpu_count, cpus_per_core)?; self.update_power_management_entry()?; + self.update_extended_feature_flags_entry()?; self.update_performance_monitoring_entry()?; self.update_brand_string_entry()?; @@ -197,6 +201,22 @@ impl super::IntelCpuid { Ok(()) } + /// Update structured extended feature flags enumeration leaf + fn update_extended_feature_flags_entry(&mut self) -> Result<(), NormalizeCpuidError> { + let leaf_7_0 = self + .get_mut(&CpuidKey::subleaf(0x7, 0)) + .ok_or(NormalizeCpuidError::MissingLeaf7)?; + + // Set FDP_EXCPTN_ONLY bit (bit 6) and ZERO_FCS_FDS bit (bit 13) as recommended in kernel + // doc. These bits are reserved in AMD. + // https://lore.kernel.org/all/20220322110712.222449-3-pbonzini@redhat.com/ + // https://github.com/torvalds/linux/commit/45016721de3c714902c6f475b705e10ae0bdd801 + set_bit(&mut leaf_7_0.result.ebx, 6, true); + set_bit(&mut leaf_7_0.result.ebx, 13, true); + + Ok(()) + } + /// Update performance monitoring entry fn update_performance_monitoring_entry(&mut self) -> Result<(), NormalizeCpuidError> { let leaf_a = self @@ -422,4 +442,30 @@ mod tests { }), ); } + + #[test] + fn test_update_extended_feature_flags_entry() { + let mut cpuid = + crate::cpu_config::x86_64::cpuid::IntelCpuid(std::collections::BTreeMap::from([( + crate::cpu_config::x86_64::cpuid::CpuidKey { + leaf: 0x7, + subleaf: 0, + }, + crate::cpu_config::x86_64::cpuid::CpuidEntry { + flags: crate::cpu_config::x86_64::cpuid::KvmCpuidFlags::SIGNIFICANT_INDEX, + ..Default::default() + }, + )])); + + cpuid.update_extended_feature_flags_entry().unwrap(); + + let leaf_7_0 = cpuid + .get(&crate::cpu_config::x86_64::cpuid::CpuidKey { + leaf: 0x7, + subleaf: 0, + }) + .unwrap(); + assert!((leaf_7_0.result.ebx & (1 << 6)) > 0); + assert!((leaf_7_0.result.ebx & (1 << 13)) > 0); + } } diff --git a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/c3.rs b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/c3.rs index b69c3177e7e..d7dda37c71d 100644 --- a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/c3.rs +++ b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/c3.rs @@ -49,7 +49,7 @@ pub fn c3() -> CustomCpuTemplate { CpuidRegisterModifier { register: CpuidRegister::Ebx, bitmap: RegisterValueFilter { - filter: 0b11111111101011111101110101111100, + filter: 0b11111111101011111101110100111100, value: 0b00000000000000000000000000000000, }, }, diff --git a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2.rs b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2.rs index e1cd8b88848..87c34e2dd36 100644 --- a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2.rs +++ b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2.rs @@ -49,7 +49,7 @@ pub fn t2() -> CustomCpuTemplate { CpuidRegisterModifier { register: CpuidRegister::Ebx, bitmap: RegisterValueFilter { - filter: 0b11111111111011111111101001010100, + filter: 0b11111111111011111101101000010100, value: 0b00000000000000000000001000000000, }, }, diff --git a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2a.rs b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2a.rs index be42190bccd..d5a6a20fd9f 100644 --- a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2a.rs +++ b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2a.rs @@ -49,7 +49,7 @@ pub fn t2a() -> CustomCpuTemplate { CpuidRegisterModifier { register: CpuidRegister::Ebx, bitmap: RegisterValueFilter { - filter: 0b11111111111011111111101001010100, + filter: 0b11111111111011111101101000010100, value: 0b00000000000000000000001000000000, }, }, diff --git a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2cl.rs b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2cl.rs index aba1174d27e..c5b121cb5af 100644 --- a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2cl.rs +++ b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2cl.rs @@ -50,7 +50,7 @@ pub fn t2cl() -> CustomCpuTemplate { CpuidRegisterModifier { register: CpuidRegister::Ebx, bitmap: RegisterValueFilter { - filter: 0b11111111111011111111101001010100, + filter: 0b11111111111011111101101000010100, value: 0b00000000000000000000001000000000, }, }, @@ -131,7 +131,7 @@ pub fn t2cl() -> CustomCpuTemplate { msr_modifiers: vec![RegisterModifier { addr: 0x10a, bitmap: RegisterValueFilter { - filter: 0b1111111111111111111111111111111111111111111111111111111111111111, + filter: 0b1111111111111111111111111111111111111111111101111111111111111011, value: 0b0000000000000000000000000000000000000000000000000000000011101011, }, }], diff --git a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2s.rs b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2s.rs index cebb8b9e20a..3dfe56f8dc1 100644 --- a/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2s.rs +++ b/src/vmm/src/cpu_config/x86_64/static_cpu_templates/t2s.rs @@ -50,7 +50,7 @@ pub fn t2s() -> CustomCpuTemplate { CpuidRegisterModifier { register: CpuidRegister::Ebx, bitmap: RegisterValueFilter { - filter: 0b11111111111011111111101001010100, + filter: 0b11111111111011111101101000010100, value: 0b00000000000000000000001000000000, }, }, @@ -132,7 +132,7 @@ pub fn t2s() -> CustomCpuTemplate { addr: 0x10a, bitmap: RegisterValueFilter { filter: 0b1111111111111111111111111111111111111111111111111111111111111111, - value: 0b0000000000000000000000000000000000000000000000000000110001001100, + value: 0b0000000000000000000000000000000000000000000010000000110001001100, }, }], } diff --git a/tests/integration_tests/build/test_coverage.py b/tests/integration_tests/build/test_coverage.py index 2b1cb35ba20..06b5fd34245 100644 --- a/tests/integration_tests/build/test_coverage.py +++ b/tests/integration_tests/build/test_coverage.py @@ -27,7 +27,7 @@ def is_on_skylake(): if utils.is_io_uring_supported(): COVERAGE_DICT = {"Intel": 83.59, "AMD": 83.18, "ARM": 83.04} else: - COVERAGE_DICT = {"Intel": 80.85, "AMD": 80.39, "ARM": 80.05} + COVERAGE_DICT = {"Intel": 80.90, "AMD": 80.39, "ARM": 80.05} PROC_MODEL = proc.proc_type() diff --git a/tests/integration_tests/functional/test_cpu_features.py b/tests/integration_tests/functional/test_cpu_features.py index 18f38b80c65..66ece763453 100644 --- a/tests/integration_tests/functional/test_cpu_features.py +++ b/tests/integration_tests/functional/test_cpu_features.py @@ -855,7 +855,6 @@ def check_masked_features(test_microvm, cpu_template): (1 << 3) | # BMI1 (1 << 4) | # HLE (1 << 5) | # AVX2 - (1 << 6) | # FPDP (1 << 8) | # BMI2 (1 << 10) | # INVPCID (1 << 11) | # RTM @@ -941,7 +940,6 @@ def check_masked_features(test_microvm, cpu_template): (0x7, 0x0, "ebx", (1 << 2) | # SGX (1 << 4) | # HLE - (1 << 6) | # FPDP (1 << 11) | # RTM (1 << 12) | # RDT_M (1 << 14) | # MPX diff --git a/tests/integration_tests/functional/test_feat_parity.py b/tests/integration_tests/functional/test_feat_parity.py index 406a15dec04..484dcf9da50 100644 --- a/tests/integration_tests/functional/test_feat_parity.py +++ b/tests/integration_tests/functional/test_feat_parity.py @@ -257,6 +257,8 @@ def test_feat_parity_msr_arch_cap(vm): (1 << 6) | # IF_PSCHANGE_MC_NO (1 << 7) # TSX_CTRL ) + if global_props.cpu_codename == "INTEL_CASCADELAKE": + expected |= (1 << 19) # RRSBA # fmt: on assert actual == expected, f"{actual=:#x} != {expected=:#x}" elif cpu_template == "T2A": diff --git a/tests/integration_tests/performance/test_versioned_serialization_benchmark.py b/tests/integration_tests/performance/test_versioned_serialization_benchmark.py deleted file mode 100644 index 67d2946fa8f..00000000000 --- a/tests/integration_tests/performance/test_versioned_serialization_benchmark.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""A test that ensures that all unit tests pass at integration time.""" - -import json -import logging -import os -import platform -import shutil -from pathlib import Path - -import pytest - -from framework import utils -from framework.defs import FC_WORKSPACE_DIR -from host_tools import proc - -BENCHMARK_DIRECTORY = "{}/src/vmm".format(FC_WORKSPACE_DIR) -DEFAULT_BUILD_TARGET = "{}-unknown-linux-musl".format(platform.machine()) - -PROC_MODEL = proc.proc_type() - -NSEC_IN_MSEC = 1000000 - -BASELINES = { - "Intel": { - "serialize": { - "no-crc": {"target": 0.195, "delta": 0.050}, # milliseconds # milliseconds - "crc": {"target": 0.244, "delta": 0.44}, # milliseconds # milliseconds - }, - "deserialize": { - "no-crc": {"target": 0.050, "delta": 0.02}, # milliseconds # milliseconds - "crc": {"target": 0.070, "delta": 0.030}, # milliseconds # milliseconds - }, - }, - "AMD": { - "serialize": { - "no-crc": {"target": 0.084, "delta": 0.05}, # milliseconds # milliseconds - "crc": {"target": 0.108, "delta": 0.025}, # milliseconds # milliseconds - }, - "deserialize": { - "no-crc": {"target": 0.030, "delta": 0.02}, # milliseconds # milliseconds - "crc": {"target": 0.052, "delta": 0.04}, # milliseconds # milliseconds - }, - }, - "ARM": { - "serialize": { - "no-crc": {"target": 0.050, "delta": 0.03}, # milliseconds # milliseconds - "crc": {"target": 0.050, "delta": 0.025}, # milliseconds # milliseconds - }, - "deserialize": { - "no-crc": {"target": 0.031, "delta": 0.02}, # milliseconds # milliseconds - "crc": {"target": 0.038, "delta": 0.02}, # milliseconds # milliseconds - }, - }, -} - - -def _check_statistics(directory, mean): - proc_model = [item for item in BASELINES if item in PROC_MODEL] - assert len(proc_model) == 1, "Could not get processor model!" - - if "deserialize" in directory.lower(): - bench = "deserialize" - else: - bench = "serialize" - - if "crc" in directory.lower(): - attribute = "crc" - else: - attribute = "no-crc" - - measure = BASELINES[proc_model[0]][bench][attribute] - target, delta = measure["target"], measure["delta"] - assert mean == pytest.approx( - target, abs=delta, rel=1e-6 - ), f"Benchmark result {directory} has changed!" - - return f"{target - delta} <= result <= {target + delta}" - - -def test_serialization_benchmark(monkeypatch, record_property): - """ - Benchmark test for MicrovmState serialization/deserialization. - - @type: performance - """ - logger = logging.getLogger("serialization_benchmark") - - # Move into the benchmark directory - monkeypatch.chdir(BENCHMARK_DIRECTORY) - - # Run benchmark test - cmd = "cargo bench --bench snapshots --target {}".format(DEFAULT_BUILD_TARGET) - result = utils.run_cmd_sync(cmd) - assert result.returncode == 0 - - # Parse each Criterion benchmark from the result folder and - # check the results against a baseline - results_dir = Path(FC_WORKSPACE_DIR) / "build/vmm_benchmark/snapshots" - for directory in os.listdir(results_dir): - # Ignore the 'report' directory as it is of no use to us - if directory == "report": - continue - - logger.info("Benchmark: %s", directory) - - # Retrieve the 'estimates.json' file content - json_file = results_dir / directory / "base/estimates.json" - estimates = json.loads(json_file.read_text()) - - # Save the Mean measurement(nanoseconds) and transform it(milliseconds) - mean = estimates["mean"]["point_estimate"] / NSEC_IN_MSEC - logger.info("Mean: %f", mean) - - criteria = _check_statistics(directory, mean) - record_property(f"{directory}_ms", mean) - record_property(f"{directory}_criteria", criteria) - - # Cleanup the Target directory - shutil.rmtree(results_dir)