From 05a9cec3596d4018ed22b934d9b8c04b7369086c Mon Sep 17 00:00:00 2001 From: pgmpablo157321 Date: Thu, 5 Oct 2023 10:20:01 -0500 Subject: [PATCH] Add parameter to enforce maximun duration --- loadgen/bindings/python_api.cc | 2 ++ loadgen/issue_query_controller.cc | 22 ++++++++++++---------- loadgen/results.cc | 8 ++++++-- loadgen/test_settings.h | 1 + loadgen/test_settings_internal.cc | 4 ++++ loadgen/test_settings_internal.h | 1 + tools/submission/submission_checker.py | 10 ++++++++++ 7 files changed, 36 insertions(+), 12 deletions(-) diff --git a/loadgen/bindings/python_api.cc b/loadgen/bindings/python_api.cc index cfe24bd3c..87b1b155e 100644 --- a/loadgen/bindings/python_api.cc +++ b/loadgen/bindings/python_api.cc @@ -312,6 +312,8 @@ PYBIND11_MODULE(mlperf_loadgen, m) { .def_readwrite("max_duration_ms", &TestSettings::max_duration_ms) .def_readwrite("min_query_count", &TestSettings::min_query_count) .def_readwrite("max_query_count", &TestSettings::max_query_count) + .def_readwrite("enforce_max_duration", + &TestSettings::enforce_max_duration) .def_readwrite("qsl_rng_seed", &TestSettings::qsl_rng_seed) .def_readwrite("sample_index_rng_seed", &TestSettings::sample_index_rng_seed) diff --git a/loadgen/issue_query_controller.cc b/loadgen/issue_query_controller.cc index 8e1e80f1a..0c0014043 100644 --- a/loadgen/issue_query_controller.cc +++ b/loadgen/issue_query_controller.cc @@ -515,19 +515,21 @@ void IssueQueryController::IssueQueriesInternal(size_t query_stride, // Checks if we have exceeded max_duration. if (settings.max_duration.count() != 0 && duration > settings.max_duration) { - LogDetail([thread_idx, duration](AsyncDetail& detail) { + if(settings.enforce_max_duration){ + LogDetail([thread_idx, duration](AsyncDetail& detail) { #if USE_NEW_LOGGING_FORMAT - std::stringstream ss; - ss << "IssueQueryThread " << thread_idx - << " Ending early: Max test duration reached." - << " duration_ns " << duration.count(); - MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); + std::stringstream ss; + ss << "IssueQueryThread " << thread_idx + << " Ending early: Max test duration reached." + << " duration_ns " << duration.count(); + MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); #else - detail.Error("IssueQueryThread ", std::to_string(thread_idx), - " Ending early: Max test duration reached.", "duration_ns", - std::to_string(duration.count())); + detail.Error("IssueQueryThread ", std::to_string(thread_idx), + " Ending early: Max test duration reached.", "duration_ns", + std::to_string(duration.count())); #endif - }); + }); + } ran_out_of_generated_queries = false; break; } diff --git a/loadgen/results.cc b/loadgen/results.cc index 445de8901..26c64a4e0 100644 --- a/loadgen/results.cc +++ b/loadgen/results.cc @@ -473,15 +473,19 @@ void PerformanceSummary::LogSummary(AsyncSummary& summary) { } bool perf_constraints_met = PerfConstraintsMet(&perf_constraints_recommendation); - bool all_constraints_met = min_duration_met && min_queries_met && + + bool all_constraints_met = min_duration_met && perf_constraints_met && early_stopping_met; + if (settings.enforce_max_duration){ + all_constraints_met = all_constraints_met && min_queries_met; + } summary("Result is : ", all_constraints_met ? "VALID" : "INVALID"); if (HasPerfConstraints()) { summary(" Performance constraints satisfied : ", perf_constraints_met ? "Yes" : "NO"); } summary(" Min duration satisfied : ", min_duration_met ? "Yes" : "NO"); - summary(" Min queries satisfied : ", min_queries_met ? "Yes" : "NO"); + summary(" Min queries satisfied : ", min_queries_met ? "Yes" : settings.enforce_max_duration? "NO" : "Skipped"); summary(" Early stopping satisfied: ", early_stopping_met ? "Yes" : "NO"); if (!all_constraints_met) { diff --git a/loadgen/test_settings.h b/loadgen/test_settings.h index b0018380d..4b1c81633 100644 --- a/loadgen/test_settings.h +++ b/loadgen/test_settings.h @@ -196,6 +196,7 @@ struct TestSettings { /**@{*/ uint64_t min_duration_ms = 10000; uint64_t max_duration_ms = 0; ///< 0: Infinity. + bool enforce_max_duration = true; uint64_t min_query_count = 100; uint64_t max_query_count = 0; ///< 0: Infinity. /**@}*/ diff --git a/loadgen/test_settings_internal.cc b/loadgen/test_settings_internal.cc index 2bcf62c29..6afd38e22 100644 --- a/loadgen/test_settings_internal.cc +++ b/loadgen/test_settings_internal.cc @@ -34,6 +34,7 @@ TestSettingsInternal::TestSettingsInternal( target_duration(std::chrono::milliseconds(requested.min_duration_ms)), min_duration(std::chrono::milliseconds(requested.min_duration_ms)), max_duration(std::chrono::milliseconds(requested.max_duration_ms)), + enforce_max_duration(requested.enforce_max_duration), min_query_count(requested.min_query_count), max_query_count(requested.max_query_count), min_sample_count(0), @@ -419,6 +420,7 @@ void TestSettingsInternal::LogEffectiveSettings() const { s.target_duration.count()); MLPERF_LOG(detail, "effective_min_duration_ms", s.min_duration.count()); MLPERF_LOG(detail, "effective_max_duration_ms", s.max_duration.count()); + MLPERF_LOG(detail, "effective_enforce_max_duration", s.enforce_max_duration); MLPERF_LOG(detail, "effective_min_query_count", s.min_query_count); MLPERF_LOG(detail, "effective_max_query_count", s.max_query_count); MLPERF_LOG(detail, "effective_min_sample_count", s.min_sample_count); @@ -658,6 +660,8 @@ int TestSettings::FromConfig(const std::string &path, const std::string &model, } lookupkv(model, scenario, "min_duration", &min_duration_ms, nullptr); lookupkv(model, scenario, "max_duration", &max_duration_ms, nullptr); + if (lookupkv(model, scenario, "enforce_max_duration", &val, nullptr)) + enforce_max_duration = (val == 1) ? true : false; lookupkv(model, scenario, "min_query_count", &min_query_count, nullptr); lookupkv(model, scenario, "max_query_count", &max_query_count, nullptr); lookupkv(model, scenario, "qsl_rng_seed", &qsl_rng_seed, nullptr); diff --git a/loadgen/test_settings_internal.h b/loadgen/test_settings_internal.h index 5222f3156..6bac52615 100644 --- a/loadgen/test_settings_internal.h +++ b/loadgen/test_settings_internal.h @@ -65,6 +65,7 @@ struct TestSettingsInternal { // duration at the end of the run. std::chrono::milliseconds min_duration{0}; std::chrono::milliseconds max_duration{0}; + bool enforce_max_duration; uint64_t min_query_count; uint64_t max_query_count; uint64_t min_sample_count; // Offline only. diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 0bae5b0d4..81cf96b2d 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -2004,6 +2004,8 @@ def check_performance_dir( min_query_count = mlperf_log["effective_min_query_count"] samples_per_query = mlperf_log["effective_samples_per_query"] min_duration = mlperf_log["effective_min_duration_ms"] + enforce_max_duration = mlperf_log.get("effective_enforce_max_duration", True) + min_queries_met = mlperf_log["result_min_queries_met"] sut_name = mlperf_log["sut_name"] else: fname = os.path.join(path, "mlperf_log_summary.txt") @@ -2026,6 +2028,7 @@ def check_performance_dir( min_query_count = int(rt["min_query_count"]) samples_per_query = int(rt["samples_per_query"]) min_duration = int(rt["min_duration (ms)"]) + enforce_max_duration = bool(rt.get("effective_enforce_max_duration", True)) if scenario == "SingleStream": qps_wo_loadgen_overhead = float(rt["QPS w/o loadgen overhead"]) sut_name = str(rt["System Under Test (SUT) name: "]) @@ -2170,6 +2173,13 @@ def check_performance_dir( min_duration, ) + if not min_queries_met and not enforce_max_duration: + log.error( + "%s Loadgen needs to enforce max duration for official submissions. Enforced %s", + fname, + enforce_max_duration, + ) + inferred = False if scenario_fixed != scenario: inferred, res = get_inferred_result(scenario_fixed, scenario, res, mlperf_log, config, True)