From 9f8c095ca2267fa63307ce913fc955de2756b55e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Jul 2025 14:18:14 +0530 Subject: [PATCH 01/15] Fix rclone config for abtf-ssd-pytorch --- script/get-ml-model-abtf-ssd-pytorch/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-ml-model-abtf-ssd-pytorch/meta.yaml b/script/get-ml-model-abtf-ssd-pytorch/meta.yaml index 54ea9e096..fa1b5d39a 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/meta.yaml +++ b/script/get-ml-model-abtf-ssd-pytorch/meta.yaml @@ -225,7 +225,7 @@ variations: enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - yes - - tags: get,rclone-config,_config-name.cognata + - tags: get,rclone-config,_config-name.mlc-cognata force_cache: true enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: @@ -257,4 +257,4 @@ tests: run_inputs: - variations_list: - onnx,rclone,mlc,dry-run - - pytorch,rclone,mlc,dry-run \ No newline at end of file + - pytorch,rclone,mlc,dry-run From b4445eb70beaeedfb2e716f24c8a39ea9318634a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Jul 2025 23:09:18 +0530 Subject: [PATCH 02/15] Fixes for mlperf-automotive submission generation --- script/generate-mlperf-inference-submission/customize.py | 3 +++ script/generate-mlperf-inference-submission/meta.yaml | 1 + script/run-mlperf-inference-app/meta.yaml | 1 + script/run-mlperf-inference-submission-checker/meta.yaml | 8 +++++++- script/submit-mlperf-results/customize.py | 8 ++------ script/submit-mlperf-results/meta.yaml | 7 +++++-- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index eb614ecd5..ad40ae6ab 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -116,6 +116,9 @@ def generate_submission(env, state, inp, submission_division, logger): if 'MLC_MLPERF_SUBMISSION_SYSTEM_TYPE' in env: system_meta_tmp['system_type'] = env['MLC_MLPERF_SUBMISSION_SYSTEM_TYPE'] + if 'MLC_MLPERF_SUBMISSION_SYSTEM_STATUS' in env: + system_meta_tmp['status'] = env['MLC_MLPERF_SUBMISSION_SYSTEM_STATUS'] + if submission_division != "": system_meta_tmp['division'] = submission_division division = submission_division diff --git a/script/generate-mlperf-inference-submission/meta.yaml b/script/generate-mlperf-inference-submission/meta.yaml index 886509958..4060c6058 100644 --- a/script/generate-mlperf-inference-submission/meta.yaml +++ b/script/generate-mlperf-inference-submission/meta.yaml @@ -59,6 +59,7 @@ docker: use_host_user_id: true input_mapping: analyzer_settings_file: MLC_MLPERF_POWER_ANALYZER_SETTINGS_FILE_PATH + status: MLC_MLPERF_SUBMISSION_SYSTEM_STATUS category: MLC_MLPERF_SUBMISSION_CATEGORY clean: MLC_MLPERF_CLEAN_SUBMISSION_DIR dashboard: MLC_MLPERF_DASHBOARD diff --git a/script/run-mlperf-inference-app/meta.yaml b/script/run-mlperf-inference-app/meta.yaml index 4ce34faaf..758d4374a 100644 --- a/script/run-mlperf-inference-app/meta.yaml +++ b/script/run-mlperf-inference-app/meta.yaml @@ -62,6 +62,7 @@ input_mapping: execution_mode: MLC_MLPERF_RUN_STYLE find_performance: MLC_MLPERF_FIND_PERFORMANCE_MODE framework: MLC_MLPERF_BACKEND + status: MLC_MLPERF_SUBMISSION_SYSTEM_STATUS docker_keep_alive: MLC_DOCKER_CONTAINER_KEEP_ALIVE get_platform_details: MLC_GET_PLATFORM_DETAILS gpu_name: MLC_NVIDIA_GPU_NAME diff --git a/script/run-mlperf-inference-submission-checker/meta.yaml b/script/run-mlperf-inference-submission-checker/meta.yaml index 3547dd2aa..99e4e9de0 100644 --- a/script/run-mlperf-inference-submission-checker/meta.yaml +++ b/script/run-mlperf-inference-submission-checker/meta.yaml @@ -111,7 +111,9 @@ post_deps: - enable_if_env: MLC_MLPERF_SUBMITTER_ID: - 'yes' - tags: submit,mlperf,results,_inference + names: + - submit-results + tags: submit,mlperf,results env: MLC_MLPERF_SUBMISSION_FILE: <<>> tags: @@ -136,6 +138,8 @@ variations: add_deps_recursive: preprocess-mlperf-submission: tags: _wg-inference + submit-results: + tags: _wg-inference deps: - names: - inference-src @@ -152,6 +156,8 @@ variations: add_deps_recursive: preprocess-mlperf-submission: tags: _wg-automotive + submit-results: + tags: _wg-automotive deps: - names: - automotive-src diff --git a/script/submit-mlperf-results/customize.py b/script/submit-mlperf-results/customize.py index db89c4655..909c8b758 100644 --- a/script/submit-mlperf-results/customize.py +++ b/script/submit-mlperf-results/customize.py @@ -37,8 +37,6 @@ def preprocess(i): signed_url = r['signed_url'] submission_id = r['submission_id'] - # print(signed_url) - # print(submission_id) r = upload_file_to_signed_url(file_path, signed_url, logger) if r['return'] > 0: return r @@ -77,16 +75,14 @@ def get_signed_url(server, benchmark, submitter_id, submitter_name, file_path): # print("Response:", response.json()) pass else: - # print(f"Request failed with status code {response.status_code}") - # print("Response:", response.text) - pass + return {"return": 1, + "error": f"HTTP status code: {response.status_code}"} except requests.exceptions.RequestException as e: return {"return": 1, "error": f"An error occurred in connecting to the server: {e}"} response_json = response.json() - # response = json.loads(response_json) try: signed_url = response_json['signed_url'] submission_id = response_json['submission_id'] diff --git a/script/submit-mlperf-results/meta.yaml b/script/submit-mlperf-results/meta.yaml index 83e2faeb2..94307f0bd 100644 --- a/script/submit-mlperf-results/meta.yaml +++ b/script/submit-mlperf-results/meta.yaml @@ -17,6 +17,9 @@ tags: - submission uid: cc01f0a82bef4216 variations: - inference: + wg-inference: env: - MLC_MLPERF_BENCHMARK: "Inference" + MLC_MLPERF_BENCHMARK: Inference + wg-automotive: + env: + MLC_MLPERF_BENCHMARK: Automotive From 1bb3068d948b329c7556385d840fd4090bbdfffb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 11 Jul 2025 17:39:44 +0000 Subject: [PATCH 03/15] [Automated Commit] Format Codebase [skip ci] --- script/generate-mlperf-inference-submission/customize.py | 2 +- script/submit-mlperf-results/customize.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index ad40ae6ab..875a12906 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -118,7 +118,7 @@ def generate_submission(env, state, inp, submission_division, logger): if 'MLC_MLPERF_SUBMISSION_SYSTEM_STATUS' in env: system_meta_tmp['status'] = env['MLC_MLPERF_SUBMISSION_SYSTEM_STATUS'] - + if submission_division != "": system_meta_tmp['division'] = submission_division division = submission_division diff --git a/script/submit-mlperf-results/customize.py b/script/submit-mlperf-results/customize.py index 909c8b758..1c2c6b619 100644 --- a/script/submit-mlperf-results/customize.py +++ b/script/submit-mlperf-results/customize.py @@ -76,7 +76,7 @@ def get_signed_url(server, benchmark, submitter_id, submitter_name, file_path): pass else: return {"return": 1, - "error": f"HTTP status code: {response.status_code}"} + "error": f"HTTP status code: {response.status_code}"} except requests.exceptions.RequestException as e: return {"return": 1, From f2ce62de693b0a563935da270e34141d79609143 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Jul 2025 06:45:27 +0530 Subject: [PATCH 04/15] Clean up submission generation for wg-automotive --- .../get-mlperf-inference-submission-dir/meta.yaml | 14 ++++++++++++-- .../meta.yaml | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/script/get-mlperf-inference-submission-dir/meta.yaml b/script/get-mlperf-inference-submission-dir/meta.yaml index 9590ef7f8..4ff61d2f6 100644 --- a/script/get-mlperf-inference-submission-dir/meta.yaml +++ b/script/get-mlperf-inference-submission-dir/meta.yaml @@ -26,13 +26,23 @@ tags: - directory uid: ddf36a41d6934a7e variations: + wg-inference: + group: working-group + default: true + env: + MLC_MLPERF_WORKING_GROUP: inference + wg-automotive: + group: working-group + env: + MLC_MLPERF_WORKING_GROUP: automotive + version.#: env: MLC_MLPERF_INFERENCE_SUBMISSION_VERSION: '#' group: version - version.4_1-dev: + version.5_1-dev: default: true env: - MLC_MLPERF_INFERENCE_SUBMISSION_VERSION: 4_1-dev + MLC_MLPERF_INFERENCE_SUBMISSION_VERSION: 5_1-dev group: version versions: {} diff --git a/script/run-mlperf-inference-submission-checker/meta.yaml b/script/run-mlperf-inference-submission-checker/meta.yaml index 99e4e9de0..aef5ca30c 100644 --- a/script/run-mlperf-inference-submission-checker/meta.yaml +++ b/script/run-mlperf-inference-submission-checker/meta.yaml @@ -140,6 +140,8 @@ variations: tags: _wg-inference submit-results: tags: _wg-inference + get-mlperf-submission-dir: + tags: _wg-inference deps: - names: - inference-src @@ -158,6 +160,8 @@ variations: tags: _wg-automotive submit-results: tags: _wg-automotive + get-mlperf-submission-dir: + tags: _wg-automotive deps: - names: - automotive-src @@ -169,6 +173,17 @@ variations: enable_if_env: MLC_MLPERF_INFERENCE_PULL_SRC_CHANGES: - 'yes' + version.master: + group: version + version.master,wg-inference: + adr: + submission-checker-src: + version: master + version.master,wg-automotive: + adr: + submission-checker-src: + version: master + versions: master: adr: From e98627d643779f7bd1f9308c1409cbb7d29d56e4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Jul 2025 07:12:22 +0530 Subject: [PATCH 05/15] Fixes for mlperf-inference-v5.1 --- script/app-mlperf-inference/meta.yaml | 28 ++++++++++++++++++- .../meta.yaml | 18 ++++++++++-- script/get-mlperf-inference-results/meta.yaml | 12 ++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 849a9ff73..41671a334 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -418,6 +418,30 @@ variations: docker: base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v5.0-cuda12.8-pytorch25.01-ubuntu24.04-aarch64-Grace-release + nvidia-original,r5.1-dev_default: + env: + MLC_NVIDIA_MITTEN_FROM_SRC: 'yes' + docker: + os_version: "24.04" + user: 'ubuntu' + build_deps: + - tags: detect,os + image_name: mlperf-inference-nvidia-v5.0-common + build_env: + ENV: release + + update_meta_if_env: + - enable_if_env: + MLC_HOST_PLATFORM_FLAVOR: + - x86_64 + docker: + base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v5.0-cuda12.8-pytorch25.01-ubuntu24.04-x86_64-release + - skip_if_env: + MLC_HOST_PLATFORM_FLAVOR: + - x86_64 + docker: + base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v5.0-cuda12.8-pytorch25.01-ubuntu24.04-aarch64-Grace-release + nvidia-original,gptj_: env: BUILD_TRTLLM: 1 @@ -2076,7 +2100,7 @@ variations: reproducibility add_deps_recursive: nvidia-inference-common-code: - tags: _mlcommons,_v5.1 + tags: _custom,_v5.1-dev nvidia-inference-server: version: r5.0 tags: _mlcommons @@ -2086,6 +2110,8 @@ variations: tags: _v4.1 nvidia-scratch-space: tags: _version.5.1-dev + pycuda: + version: "2024.1" default_env: MLC_SKIP_SYS_UTILS: 'yes' MLC_REGENERATE_MEASURE_FILES: 'yes' diff --git a/script/get-mlperf-inference-nvidia-common-code/meta.yaml b/script/get-mlperf-inference-nvidia-common-code/meta.yaml index c25fc7859..68d303ca2 100644 --- a/script/get-mlperf-inference-nvidia-common-code/meta.yaml +++ b/script/get-mlperf-inference-nvidia-common-code/meta.yaml @@ -20,21 +20,35 @@ tags: - common-code uid: 26b78bf3ffdc4926 variations: - ctuning: - group: repo-owner custom: group: repo-owner + adr: + mlperf-inference-results: + tags: _custom go: group: repo-owner + adr: + mlperf-inference-results: + tags: _go mlcommons: default: true group: repo-owner + adr: + mlperf-inference-results: + tags: _mlcommons nvidia-only: group: repo-owner + adr: + mlperf-inference-results: + tags: _nvidia-only v5.0: add_deps_recursive: mlperf-inference-results: tags: _code-only,_v5.0 + v5.1-dev: + add_deps_recursive: + mlperf-inference-results: + tags: _code-only,_v5.1-dev versions: r2.1: add_deps_recursive: diff --git a/script/get-mlperf-inference-results/meta.yaml b/script/get-mlperf-inference-results/meta.yaml index 9b42fdf2a..d3b886de3 100644 --- a/script/get-mlperf-inference-results/meta.yaml +++ b/script/get-mlperf-inference-results/meta.yaml @@ -42,6 +42,11 @@ variations: inference-results-repo: tags: _branch.mlc-code-only-for-v5.0 group: repo-branch + code-only-for-v5.1: + adr: + inference-results-repo: + tags: _branch.mlc-code-only-for-v5.1 + group: repo-branch ctuning: env: GITHUB_REPO_OWNER: ctuning @@ -65,10 +70,17 @@ variations: NVIDIA_ONLY: 'yes' group: source-repo v5.0: + group: version env: MLC_GIT_URL: https://github.com/<<>>/inference_results_v5.0.git MLC_MLPERF_INFERENCE_RESULTS_VERSION_NAME: v5.0 MLC_VERSION: "v5.0" + v5.1-dev: + group: version + env: + MLC_GIT_URL: https://github.com/<<>>/inference_results_v5.0.git + MLC_MLPERF_INFERENCE_RESULTS_VERSION_NAME: v5.1-dev + MLC_VERSION: "v5.0" versions: v2.1: env: From f8bd359542024b2f721c60bc341b4000193ea63c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Jul 2025 07:25:16 +0530 Subject: [PATCH 06/15] Fix nvidia-mitten sha for v5.0 --- script/app-mlperf-inference/meta.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 41671a334..04c57c7b7 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -2088,6 +2088,8 @@ variations: version: r5.0 nvidia-scratch-space: tags: _version.5.0 + nvidia-mitten-git-src: + tags: _sha.e5574192b3ae76910b6824a9275d14a4158d8afa pycuda: version: "2024.1" default_env: @@ -2110,6 +2112,8 @@ variations: tags: _v4.1 nvidia-scratch-space: tags: _version.5.1-dev + nvidia-mitten-git-src: + tags: _sha.e5574192b3ae76910b6824a9275d14a4158d8afa pycuda: version: "2024.1" default_env: From 799ef1ba1ec0c3bccfbc4518c506636c66238fde Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Jul 2025 07:34:02 +0530 Subject: [PATCH 07/15] Fix nvidia-inference-server version for v5.1 --- script/app-mlperf-inference/meta.yaml | 2 +- script/build-mlperf-inference-server-nvidia/meta.yaml | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 04c57c7b7..3716f6b88 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -2105,7 +2105,7 @@ variations: tags: _custom,_v5.1-dev nvidia-inference-server: version: r5.0 - tags: _mlcommons + tags: _custom nvidia-harness: tags: _v5.0 intel-harness: diff --git a/script/build-mlperf-inference-server-nvidia/meta.yaml b/script/build-mlperf-inference-server-nvidia/meta.yaml index fb345bc1f..6e801b536 100644 --- a/script/build-mlperf-inference-server-nvidia/meta.yaml +++ b/script/build-mlperf-inference-server-nvidia/meta.yaml @@ -161,12 +161,6 @@ variations: MLC_MLPERF_DEVICE: cuda MLC_MLPERF_DEVICE_LIB_NAMESPEC: cudart - ctuning: - group: code - default: true - add_deps_recursive: - nvidia-inference-common-code: - tags: _ctuning go: group: code add_deps_recursive: From 8cb2db8fe068d3a381119013742f04beca2fd818 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Jul 2025 15:25:07 +0530 Subject: [PATCH 08/15] Fix nvidia mlperf-inference 5.1-dev --- script/build-mlperf-inference-server-nvidia/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/build-mlperf-inference-server-nvidia/run.sh b/script/build-mlperf-inference-server-nvidia/run.sh index c5f4e1a8a..5052e59bc 100644 --- a/script/build-mlperf-inference-server-nvidia/run.sh +++ b/script/build-mlperf-inference-server-nvidia/run.sh @@ -12,8 +12,8 @@ if [[ ${MLC_MLPERF_DEVICE} == "inferentia" ]]; then make prebuild fi -# Perform sed replacement only if version is 5.0 -if [[ "${MLC_MLPERF_INFERENCE_VERSION}" == "5.0" ]]; then + +if [[ "${MLC_MLPERF_INFERENCE_VERSION}" =~ ^[5-9]\.[0-9]+$ ]]; then echo "Replacing /work/ with ${MLC_MLPERF_INFERENCE_NVIDIA_CODE_PATH} in all files..." find . -type f -exec sed -i "s|/work/|${MLC_MLPERF_INFERENCE_NVIDIA_CODE_PATH}/|g" {} + fi From c1c36a2712b9049d21e04b689db96e6c10328aeb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Jul 2025 15:34:36 +0530 Subject: [PATCH 09/15] Fix nvidia mlperf-inference 5.1-dev --- script/build-mlperf-inference-server-nvidia/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-mlperf-inference-server-nvidia/run.sh b/script/build-mlperf-inference-server-nvidia/run.sh index 5052e59bc..3c5145464 100644 --- a/script/build-mlperf-inference-server-nvidia/run.sh +++ b/script/build-mlperf-inference-server-nvidia/run.sh @@ -13,7 +13,7 @@ if [[ ${MLC_MLPERF_DEVICE} == "inferentia" ]]; then fi -if [[ "${MLC_MLPERF_INFERENCE_VERSION}" =~ ^[5-9]\.[0-9]+$ ]]; then +if [[ "${MLC_MLPERF_INFERENCE_VERSION}" =~ ^[5-9]\.[0-9]+(-dev)?$ ]]; then echo "Replacing /work/ with ${MLC_MLPERF_INFERENCE_NVIDIA_CODE_PATH} in all files..." find . -type f -exec sed -i "s|/work/|${MLC_MLPERF_INFERENCE_NVIDIA_CODE_PATH}/|g" {} + fi From 5b5d6eb7d9f231ff8f3ba183c5c310510b2b92ae Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Jul 2025 15:47:39 +0530 Subject: [PATCH 10/15] Fix nvidia mlperf-inference 5.1-dev --- script/add-custom-nvidia-system/meta.yaml | 11 ++++++----- script/build-mlperf-inference-server-nvidia/meta.yaml | 3 --- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/script/add-custom-nvidia-system/meta.yaml b/script/add-custom-nvidia-system/meta.yaml index ec7202857..94285d986 100644 --- a/script/add-custom-nvidia-system/meta.yaml +++ b/script/add-custom-nvidia-system/meta.yaml @@ -78,6 +78,7 @@ deps: enable_if_env: MLC_MLPERF_INFERENCE_VERSION: - "5.0" + - "5.1-dev" variations: nvidia-only: @@ -95,11 +96,6 @@ variations: add_deps_recursive: nvidia-inference-common-code: tags: _mlcommons - ctuning: - group: code - add_deps_recursive: - nvidia-inference-common-code: - tags: _ctuning go: group: code add_deps_recursive: @@ -109,9 +105,14 @@ variations: add_deps_recursive: nvidia-inference-common-code: tags: _v5.0 + v5.1-dev: + add_deps_recursive: + nvidia-inference-common-code: + tags: _v5.1-dev +#deprecated versions: r2.1: add_deps_recursive: diff --git a/script/build-mlperf-inference-server-nvidia/meta.yaml b/script/build-mlperf-inference-server-nvidia/meta.yaml index 6e801b536..014a68001 100644 --- a/script/build-mlperf-inference-server-nvidia/meta.yaml +++ b/script/build-mlperf-inference-server-nvidia/meta.yaml @@ -137,12 +137,9 @@ post_deps: - tags: add,custom,system,nvidia names: - custom-system-nvidia - - nvidia-inference-common-code skip_if_env: MLC_CUSTOM_SYSTEM_NVIDIA: - "no" - - False - - "False" variations: # Target devices From 9146284222d2697716a2726a93a7f33580cea2f4 Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 12 Jul 2025 18:53:45 +0100 Subject: [PATCH 11/15] Fixes for power runs --- script/get-mlperf-inference-utils/mlperf_utils.py | 3 ++- script/run-all-mlperf-models/run-mobilenet-models.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/script/get-mlperf-inference-utils/mlperf_utils.py b/script/get-mlperf-inference-utils/mlperf_utils.py index 1e60cafc1..ffb0d3d7a 100644 --- a/script/get-mlperf-inference-utils/mlperf_utils.py +++ b/script/get-mlperf-inference-utils/mlperf_utils.py @@ -52,7 +52,8 @@ def get_result_from_log(version, model, scenario, if has_power: power_valid, power_metric, scenario, avg_power_efficiency = checker.get_power_metric( config, scenario, result_path, True, result_) - power_result = f"{round(power_metric,3)},{round(avg_power_efficiency,3)}" + if power_valid: + power_result = f"{round(power_metric,3)},{round(avg_power_efficiency,3)}" valid['power'] = power_valid elif mode == "accuracy" and os.path.exists(os.path.join(result_path, 'accuracy.txt')): diff --git a/script/run-all-mlperf-models/run-mobilenet-models.sh b/script/run-all-mlperf-models/run-mobilenet-models.sh index ad24ed87d..849bb48ec 100644 --- a/script/run-all-mlperf-models/run-mobilenet-models.sh +++ b/script/run-all-mlperf-models/run-mobilenet-models.sh @@ -22,7 +22,7 @@ function run() { exit_if_error fi } -POWER=" --power=yes --adr.mlperf-power-client.power_server=192.168.0.15 --adr.mlperf-power-client.port=4940 " +POWER=" --power=yes --adr.mlperf-power-client.power_server=192.168.0.15 --adr.mlperf-power-client.port=4940 --adr.mlperf-power-client.ntp_server=time.google.com " POWER="" #extra_option=" --minimize_disk_usage=yes" extra_option=" --minimize_disk_usage=no" From 09c3336fea1ab5b02b773762711943acaf0ca3a6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 12 Jul 2025 19:14:50 +0100 Subject: [PATCH 12/15] Update customize.py --- script/run-mlperf-power-server/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/run-mlperf-power-server/customize.py b/script/run-mlperf-power-server/customize.py index 59056a0d2..167ffc5ac 100644 --- a/script/run-mlperf-power-server/customize.py +++ b/script/run-mlperf-power-server/customize.py @@ -66,8 +66,8 @@ def preprocess(i): with open('tmp-power-server.conf', 'w') as configfile: config.write(configfile) - logger.info( - f"{section: dict(config[section]) for section in config.sections()}") + config_dict = {section: dict(config[section]) for section in config.sections()} + logger.info(f"{config_dict}") if env['MLC_HOST_OS_TYPE'] == "windows": cmd_prefix = "" From 9eeb2c10ae003dac496b62a56783fd6b16d0997b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 12 Jul 2025 18:15:01 +0000 Subject: [PATCH 13/15] [Automated Commit] Format Codebase [skip ci] --- script/run-mlperf-power-server/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/run-mlperf-power-server/customize.py b/script/run-mlperf-power-server/customize.py index 167ffc5ac..3c6c615e7 100644 --- a/script/run-mlperf-power-server/customize.py +++ b/script/run-mlperf-power-server/customize.py @@ -66,7 +66,8 @@ def preprocess(i): with open('tmp-power-server.conf', 'w') as configfile: config.write(configfile) - config_dict = {section: dict(config[section]) for section in config.sections()} + config_dict = {section: dict(config[section]) + for section in config.sections()} logger.info(f"{config_dict}") if env['MLC_HOST_OS_TYPE'] == "windows": From cde02078bec63c4cf44e7ceac614d81e05f7af8d Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 12 Jul 2025 22:31:45 +0000 Subject: [PATCH 14/15] Fixes for nvidia mlperf inference v5.1 --- script/app-mlperf-inference-nvidia/meta.yaml | 227 ++++++++++++------ .../meta.yaml | 2 +- 2 files changed, 148 insertions(+), 81 deletions(-) diff --git a/script/app-mlperf-inference-nvidia/meta.yaml b/script/app-mlperf-inference-nvidia/meta.yaml index 9a1fcb8c4..b152945ed 100644 --- a/script/app-mlperf-inference-nvidia/meta.yaml +++ b/script/app-mlperf-inference-nvidia/meta.yaml @@ -19,6 +19,9 @@ tags: - harness - nvidia-harness - nvidia + +new_env_keys: + - +LD_LIBRARY_PATH # Default environment default_env: @@ -348,9 +351,13 @@ post_deps: # Variations to customize dependencies variations: - pre5.0: {} - # MLPerf inference version + pre5.0: + group: batchsize-format-change + v5.0+: + group: batchsize-format-change v5.0: + base: + - v5.0+ group: version env: MLC_MLPERF_INFERENCE_CODE_VERSION: "v5.0" @@ -1231,10 +1238,30 @@ variations: MLC_MLPERF_NVIDIA_HARNESS_DISABLE_BETA1_SMALLK: yes SKIP_POLICIES: '0' - singlestream,run_harness: + singlestream,pre5.0: default_variations: batch-size: batch_size.1 + singlestream,v5.0+,resnet50: + default_variations: + batch-size: batch_size.resnet:1 + + singlestream,v5.0+,3d-unet_: + default_variations: + batch-size: batch_size.3d-unet:1 + + singlestream,v5.0+,sdxl: + default_variations: + batch-size: batch_size."clip1:2,clip2:2,unet:2,vae:1" + + singlestream,v5.0+,bert_: + default_variations: + batch-size: batch_size.bert:1 + + singlestream,v5.0+,retinanet: + default_variations: + batch-size: batch_size.retinanet:1 + llama2-70b_,run_harness: env: MLC_MLPERF_NVIDIA_HARNESS_USE_FP8: 'True' @@ -1256,7 +1283,7 @@ variations: default_variations: batch-size: batch_size.896 - gpu_memory.80,v5.0,num-gpus.2,llama2-70b,offline,run_harness: + gpu_memory.80,v5.0+,num-gpus.2,llama2-70b,offline,run_harness: default_variations: batch-size: batch_size."llama2-70b:1024" @@ -1264,7 +1291,7 @@ variations: default_variations: batch-size: batch_size.4 - gpu_memory.16,v5.0,gptj_,offline,run_harness: + gpu_memory.16,v5.0+,gptj_,offline,run_harness: default_variations: batch-size: batch_size."gptj:4" @@ -1272,7 +1299,7 @@ variations: default_variations: batch-size: batch_size.7 - gpu_memory.24,v5.0,gptj_,offline,run_harness: + gpu_memory.24,v5.0+,gptj_,offline,run_harness: default_variations: batch-size: batch_size."gptj:7" @@ -1280,7 +1307,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.32,v5.0,gptj_,offline,run_harness: + gpu_memory.32,v5.0+,gptj_,offline,run_harness: default_variations: batch-size: batch_size."gptj:8" @@ -1288,7 +1315,7 @@ variations: default_variations: batch-size: batch_size.14 - gpu_memory.48,v5.0,gptj_,offline,run_harness: + gpu_memory.48,v5.0+,gptj_,offline,run_harness: default_variations: batch-size: batch_size."gptj:14" @@ -1296,7 +1323,7 @@ variations: default_variations: batch-size: batch_size.10 - gpu_memory.40,v5.0,gptj_,offline,run_harness: + gpu_memory.40,v5.0+,gptj_,offline,run_harness: default_variations: batch-size: batch_size."gptj:10" @@ -1304,7 +1331,7 @@ variations: default_variations: batch-size: batch_size.32 - gpu_memory.80,v5.0,gptj_,offline,run_harness: + gpu_memory.80,v5.0+,gptj_,offline,run_harness: default_variations: batch-size: batch_size."gptj:80" @@ -1312,7 +1339,7 @@ variations: default_variations: batch-size: batch_size.2 - gpu_memory.16,v5.0,sdxl,offline,run_harness: + gpu_memory.16,v5.0+,sdxl,offline,run_harness: default_variations: batch-size: batch_size."clip1:2##clip2:2##unet:2##vae:1" @@ -1320,7 +1347,7 @@ variations: default_variations: batch-size: batch_size.2 - gpu_memory.24,v5.0,sdxl,offline,run_harness: + gpu_memory.24,v5.0+,sdxl,offline,run_harness: default_variations: batch-size: batch_size."clip1:2##clip2:2##unet:2##vae:1" @@ -1328,7 +1355,7 @@ variations: default_variations: batch-size: batch_size.3 - gpu_memory.32,v5.0,sdxl,offline,run_harness: + gpu_memory.32,v5.0+,sdxl,offline,run_harness: default_variations: batch-size: batch_size."clip1:1##clip2:2##unet:2##vae:1" @@ -1336,7 +1363,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.80,v5.0,sdxl,offline,run_harness: + gpu_memory.80,v5.0+,sdxl,offline,run_harness: default_variations: batch-size: batch_size."clip1:64##clip2:64##unet:64##vae:8" @@ -1344,7 +1371,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.96,v5.0,sdxl,offline,run_harness: + gpu_memory.96,v5.0+,sdxl,offline,run_harness: default_variations: batch-size: batch_size."clip1:64##clip2:64##unet:64##vae:8" @@ -1352,7 +1379,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.96,v5.0,sdxl,server,run_harness: + gpu_memory.96,v5.0+,sdxl,server,run_harness: default_variations: batch-size: batch_size."clip1:64##clip2:64##unet:64##vae:8" @@ -1360,7 +1387,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.80,v5.0,sdxl,server,run_harness: + gpu_memory.80,v5.0+,sdxl,server,run_harness: default_variations: batch-size: batch_size."clip1:64##clip2:64##unet:64##vae:8" @@ -1368,45 +1395,73 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.140,v5.0,sdxl,offline,run_harness: + gpu_memory.140,v5.0+,sdxl,offline,run_harness: default_variations: batch-size: batch_size."clip1:64##clip2:64##unet:64##vae:8" - gpu_memory.8,bert_,offline,run_harness: + gpu_memory.8,pre5.0,bert_,offline,run_harness: default_variations: batch-size: batch_size.256 - gpu_memory.16,bert_,offline,run_harness: + gpu_memory.8,v5.0+,bert_,offline,run_harness: + default_variations: + batch-size: batch_size.bert:256 + + gpu_memory.16,pre5.0,bert_,offline,run_harness: default_variations: batch-size: batch_size.256 - gpu_memory.24,bert_,offline,run_harness: + gpu_memory.16,v5.0+,bert_,offline,run_harness: + default_variations: + batch-size: batch_size.bert:256 + + gpu_memory.24,pre5.0,bert_,offline,run_harness: default_variations: batch-size: batch_size.256 - gpu_memory.32,bert_,offline,run_harness: + gpu_memory.24,v5.0+,bert_,offline,run_harness: + default_variations: + batch-size: batch_size."bert:256" + + gpu_memory.32,pre5.0,bert_,offline,run_harness: default_variations: batch-size: batch_size.256 - gpu_memory.48,bert_,offline,run_harness: + gpu_memory.32,v5.0+,bert_,offline,run_harness: + default_variations: + batch-size: batch_size.256 + + gpu_memory.48,pre5.0,bert_,offline,run_harness: default_variations: batch-size: batch_size.1024 - gpu_memory.40,bert_,offline,run_harness: + gpu_memory.48,v5.0+,bert_,offline,run_harness: + default_variations: + batch-size: batch_size.1024 + + gpu_memory.40,pre5.0,bert_,offline,run_harness: + default_variations: + batch-size: batch_size.256 + + gpu_memory.40,v5.0+,bert_,offline,run_harness: default_variations: batch-size: batch_size.256 - gpu_memory.80,bert_,server,run_harness: + gpu_memory.80,pre5.0,bert_,server,run_harness: default_variations: batch-size: batch_size.64 + gpu_memory.80,v5.0+,bert_,server,run_harness: + default_variations: + batch-size: batch_size.bert:64 + gpu_memory.8,pre5.0,resnet50,offline,run_harness: default_variations: batch-size: batch_size.64 env: MLC_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: "4" - gpu_memory.8,v5.0,resnet50,offline,run_harness: + gpu_memory.8,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:64" env: @@ -1418,7 +1473,7 @@ variations: env: MLC_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: "4" - gpu_memory.16,v5.0,resnet50,offline,run_harness: + gpu_memory.16,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:1024" env: @@ -1428,7 +1483,7 @@ variations: default_variations: batch-size: batch_size.64 - gpu_memory.24,v5.0,resnet50,offline,run_harness: + gpu_memory.24,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:64" @@ -1436,7 +1491,7 @@ variations: default_variations: batch-size: batch_size.2048 - gpu_memory.32,v5.0,resnet50,offline,run_harness: + gpu_memory.32,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:2048" @@ -1444,7 +1499,7 @@ variations: default_variations: batch-size: batch_size.2048 - gpu_memory.40,v5.0,resnet50,offline,run_harness: + gpu_memory.40,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:2048" @@ -1452,7 +1507,7 @@ variations: default_variations: batch-size: batch_size.2048 - gpu_memory.48,v5.0,resnet50,offline,run_harness: + gpu_memory.48,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:2048" @@ -1460,7 +1515,7 @@ variations: default_variations: batch-size: batch_size.2048 - gpu_memory.80,v5.0,resnet50,offline,run_harness: + gpu_memory.80,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:2048" @@ -1479,23 +1534,35 @@ variations: default_variations: batch-size: batch_size.64 - resnet50,multistream,run_harness,num-gpus.1: + resnet50,multistream,pre5.0,run_harness,num-gpus.1: default_variations: batch-size: batch_size.8 - resnet50,multistream,run_harness,num-gpus.2: + resnet50,multistream,v5.0+,run_harness,num-gpus.1: + default_variations: + batch-size: batch_size.resnet50:8 + + resnet50,multistream,pre5.0,run_harness,num-gpus.2: default_variations: batch-size: batch_size.4 - retinanet,multistream,run_harness: + resnet50,multistream,v5.0+,run_harness,num-gpus.2: + default_variations: + batch-size: batch_size.resnet50:4 + + retinanet,multistream,pre5.0,run_harness: default_variations: batch-size: batch_size.2 + retinanet,multistream,v5.0+,run_harness: + default_variations: + batch-size: batch_size.retinanet:2 + gpu_memory.8,pre5.0,retinanet,offline,run_harness: default_variations: batch-size: batch_size.2 - gpu_memory.8,v5.0,retinanet,offline,run_harness: + gpu_memory.8,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:2" @@ -1503,7 +1570,7 @@ variations: default_variations: batch-size: batch_size.2 - gpu_memory.16,v5.0,retinanet,offline,run_harness: + gpu_memory.16,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:2" @@ -1514,7 +1581,7 @@ variations: MLC_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: "2" MLC_MLPERF_NVIDIA_HARNESS_GPU_INFERENCE_STREAMS: "2" - gpu_memory.24,v5.0,retinanet,offline,run_harness: + gpu_memory.24,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:2" env: @@ -1525,7 +1592,7 @@ variations: default_variations: batch-size: batch_size.4 - gpu_memory.32,v5.0,retinanet,offline,run_harness: + gpu_memory.32,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:4" @@ -1533,7 +1600,7 @@ variations: default_variations: batch-size: batch_size.4 - gpu_memory.40,v5.0,retinanet,offline,run_harness: + gpu_memory.40,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:4" @@ -1542,7 +1609,7 @@ variations: default_variations: batch-size: batch_size.4 - gpu_memory.48,v5.0,retinanet,offline,run_harness: + gpu_memory.48,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:4" @@ -1550,7 +1617,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.80,v5.0,retinanet,offline,run_harness: + gpu_memory.80,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:8" @@ -1593,7 +1660,7 @@ variations: default_variations: batch-size: batch_size.4 - gpu_memory.8,v5.0,3d-unet_,offline,run_harness: + gpu_memory.8,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:4" @@ -1601,7 +1668,7 @@ variations: default_variations: batch-size: batch_size.4 - gpu_memory.16,v5.0,3d-unet_,offline,run_harness: + gpu_memory.16,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:4" @@ -1609,7 +1676,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.24,v5.0,3d-unet_,offline,run_harness: + gpu_memory.24,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:8" @@ -1617,7 +1684,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.32,v5.0,3d-unet_,offline,run_harness: + gpu_memory.32,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:8" @@ -1625,7 +1692,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.40,v5.0,3d-unet_,offline,run_harness: + gpu_memory.40,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:8" @@ -1633,7 +1700,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.48,v5.0,3d-unet_,offline,run_harness: + gpu_memory.48,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:8" @@ -1641,7 +1708,7 @@ variations: default_variations: batch-size: batch_size.8 - gpu_memory.80,v5.0,3d-unet_,offline,run_harness: + gpu_memory.80,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:8" @@ -1649,7 +1716,7 @@ variations: default_variations: batch-size: batch_size.1400 - gpu_memory.16,v5.0,dlrm_,offline,run_harness: + gpu_memory.16,v5.0+,dlrm_,offline,run_harness: default_variations: batch-size: batch_size."dlrm:1400" @@ -1659,7 +1726,7 @@ variations: env: MLC_MLPERF_NVIDIA_HARNESS_EMBEDDING_WEIGHTS_ON_GPU_PART: "0.30" - gpu_memory.24,v5.0,dlrm_,offline,run_harness: + gpu_memory.24,v5.0+,dlrm_,offline,run_harness: default_variations: batch-size: batch_size."dlrm:1400" env: @@ -1669,7 +1736,7 @@ variations: default_variations: batch-size: batch_size.1400 - gpu_memory.32,v5.0,dlrm_,offline,run_harness: + gpu_memory.32,v5.0+,dlrm_,offline,run_harness: default_variations: batch-size: batch_size."dlrm:1400" @@ -1679,7 +1746,7 @@ variations: env: MLC_MLPERF_NVIDIA_HARNESS_EMBEDDING_WEIGHTS_ON_GPU_PART: "0.40" - gpu_memory.40,v5.0,dlrm_,offline,run_harness: + gpu_memory.40,v5.0+,dlrm_,offline,run_harness: default_variations: batch-size: batch_size."dlrm:1400" env: @@ -1691,7 +1758,7 @@ variations: env: MLC_MLPERF_NVIDIA_HARNESS_EMBEDDING_WEIGHTS_ON_GPU_PART: "0.50" - gpu_memory.48,v5.0,dlrm_,offline,run_harness: + gpu_memory.48,v5.0+,dlrm_,offline,run_harness: default_variations: batch-size: batch_size."dlrm:1400" env: @@ -1701,7 +1768,7 @@ variations: default_variations: batch-size: batch_size.1400 - gpu_memory.80,v5.0,dlrm_,offline,run_harness: + gpu_memory.80,v5.0+,dlrm_,offline,run_harness: default_variations: batch-size: batch_size."dlrm:1400" @@ -1801,7 +1868,7 @@ variations: env: MLC_NVIDIA_CUSTOM_GPU: "yes" - rtx_a6000,v5.0,sdxl,offline,run_harness,batch_size.1: + rtx_a6000,v5.0+,sdxl,offline,run_harness,batch_size.1: default_variations: batch-size: batch_size."clip1:2##clip2:2##unet:2##vae:1" @@ -1809,7 +1876,7 @@ variations: default_variations: batch-size: batch_size.64 - rtx_a6000,v5.0,resnet50,offline,run_harness: + rtx_a6000,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:64" @@ -1817,7 +1884,7 @@ variations: default_variations: batch-size: batch_size.32 - rtx_a6000,v5.0,resnet50,server,run_harness: + rtx_a6000,v5.0+,resnet50,server,run_harness: default_variations: batch-size: batch_size."resnet50:32" @@ -1825,7 +1892,7 @@ variations: default_variations: batch-size: batch_size.2 - rtx_a6000,v5.0,retinanet,offline,run_harness: + rtx_a6000,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:2" @@ -1833,7 +1900,7 @@ variations: default_variations: batch-size: batch_size.2 - rtx_a6000,v5.0,retinanet,server,run_harness: + rtx_a6000,v5.0+,retinanet,server,run_harness: default_variations: batch-size: batch_size."retinanet:2" @@ -1849,7 +1916,7 @@ variations: default_variations: batch-size: batch_size.8 - rtx_a6000,v5.0,3d-unet_,offline,run_harness: + rtx_a6000,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:8" @@ -1857,7 +1924,7 @@ variations: default_variations: batch-size: batch_size.8 - rtx_a6000,v5.0,3d-unet_,server,run_harness: + rtx_a6000,v5.0+,3d-unet_,server,run_harness: default_variations: batch-size: batch_size."3d-unet:8" @@ -1873,7 +1940,7 @@ variations: default_variations: batch-size: batch_size.1400 - rtx_a6000,v5.0,dlrm_,offline,run_harness: + rtx_a6000,v5.0+,dlrm_,offline,run_harness: default_variations: batch-size: batch_size."dlrm-v2:1400" @@ -1886,7 +1953,7 @@ variations: default_variations: batch-size: batch_size.64 - rtx_a6000_ada,v5.0,resnet50,offline,run_harness: + rtx_a6000_ada,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:64" @@ -1894,7 +1961,7 @@ variations: default_variations: batch-size: batch_size.32 - rtx_a6000_ada,v5.0,resnet50,server,run_harness: + rtx_a6000_ada,v5.0+,resnet50,server,run_harness: default_variations: batch-size: batch_size."resnet50:32" @@ -1902,7 +1969,7 @@ variations: default_variations: batch-size: batch_size.2 - rtx_a6000_ada,v5.0,retinanet,offline,run_harness: + rtx_a6000_ada,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:2" @@ -1910,7 +1977,7 @@ variations: default_variations: batch-size: batch_size.2 - rtx_a6000_ada,v5.0,retinanet,server,run_harness: + rtx_a6000_ada,v5.0+,retinanet,server,run_harness: default_variations: batch-size: batch_size."retinanet:2" @@ -1926,7 +1993,7 @@ variations: default_variations: batch-size: batch_size.8 - rtx_a6000_ada,v5.0,3d-unet_,offline,run_harness: + rtx_a6000_ada,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:8" @@ -1934,7 +2001,7 @@ variations: default_variations: batch-size: batch_size.8 - rtx_a6000_ada,v5.0,3d-unet_,offline,run_harness: + rtx_a6000_ada,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:8" @@ -1950,7 +2017,7 @@ variations: default_variations: batch-size: batch_size.1400 - rtx_a6000_ada,v5.0,dlrm_,offline,run_harness: + rtx_a6000_ada,v5.0+,dlrm_,offline,run_harness: default_variations: batch-size: batch_size."dlrm-v2:1400" @@ -1968,7 +2035,7 @@ variations: default_variations: batch-size: batch_size.1 - l4,v5.0,sdxl,offline,run_harness: + l4,v5.0+,sdxl,offline,run_harness: default_variations: batch-size: batch_size."clip1:2##clip2:2##unet:2##vae:1" @@ -1981,7 +2048,7 @@ variations: default_variations: batch-size: batch_size.1 - l4,v5.0,sdxl,offline,run_harness,num-gpu.8: + l4,v5.0+,sdxl,offline,run_harness,num-gpu.8: default_variations: batch-size: batch_size."clip1:2##clip2:2##unet:2##vae:1" @@ -1995,7 +2062,7 @@ variations: default_variations: batch-size: batch_size.1 - l4,v5.0,sdxl,server,run_harness,num-gpu.1: + l4,v5.0+,sdxl,server,run_harness,num-gpu.1: default_variations: batch-size: batch_size."clip1:2;clip2:2;unet:2;vae:1" @@ -2009,7 +2076,7 @@ variations: default_variations: batch-size: batch_size.1 - l4,v5.0,sdxl,server,run_harness,num-gpu.8: + l4,v5.0+,sdxl,server,run_harness,num-gpu.8: default_variations: batch-size: batch_size."clip1:2##clip2:2##unet:2##vae:1" @@ -2030,7 +2097,7 @@ variations: default_variations: batch-size: batch_size.32 - l4,v5.0,resnet50,offline,run_harness: + l4,v5.0+,resnet50,offline,run_harness: default_variations: batch-size: batch_size."resnet50:32" @@ -2047,7 +2114,7 @@ variations: default_variations: batch-size: batch_size.16 - l4,v5.0,resnet50,server,run_harness: + l4,v5.0+,resnet50,server,run_harness: default_variations: batch-size: batch_size."resnet50:16" @@ -2055,7 +2122,7 @@ variations: default_variations: batch-size: batch_size.2 - l4,v5.0,retinanet,offline,run_harness: + l4,v5.0+,retinanet,offline,run_harness: default_variations: batch-size: batch_size."retinanet:2" @@ -2071,7 +2138,7 @@ variations: default_variations: batch-size: batch_size.2 - l4,v5.0,retinanet,server,run_harness: + l4,v5.0+,retinanet,server,run_harness: default_variations: batch-size: batch_size."retinanet:2" @@ -2092,7 +2159,7 @@ variations: default_variations: batch-size: batch_size.1 - l4,v5.0,3d-unet_,offline,run_harness: + l4,v5.0+,3d-unet_,offline,run_harness: default_variations: batch-size: batch_size."3d-unet:1" @@ -2112,7 +2179,7 @@ variations: default_variations: batch-size: batch_size.1400 - l4,v5.0,dlrm_,offline,run_harness: + l4,v5.0+,dlrm_,offline,run_harness: default_variations: batch-size: batch_size."dlrm-v2:1400" diff --git a/script/get-mlperf-inference-nvidia-common-code/meta.yaml b/script/get-mlperf-inference-nvidia-common-code/meta.yaml index 68d303ca2..e5d4a42dc 100644 --- a/script/get-mlperf-inference-nvidia-common-code/meta.yaml +++ b/script/get-mlperf-inference-nvidia-common-code/meta.yaml @@ -48,7 +48,7 @@ variations: v5.1-dev: add_deps_recursive: mlperf-inference-results: - tags: _code-only,_v5.1-dev + tags: _code-only-for-v5.1,_v5.1-dev versions: r2.1: add_deps_recursive: From 7e0783c8c24f1ae2d809b6e7aa4d06292fea3efb Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 13 Jul 2025 04:07:01 +0530 Subject: [PATCH 15/15] Dont use onnxruntime as default framework --- script/run-mlperf-inference-app/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/script/run-mlperf-inference-app/meta.yaml b/script/run-mlperf-inference-app/meta.yaml index 758d4374a..1456e3ec4 100644 --- a/script/run-mlperf-inference-app/meta.yaml +++ b/script/run-mlperf-inference-app/meta.yaml @@ -32,7 +32,6 @@ predeps: False default_env: MLC_MLPERF_IMPLEMENTATION: reference MLC_MLPERF_MODEL: resnet50 - MLC_MLPERF_BACKEND: onnxruntime MLC_MLPERF_DEVICE: cpu MLC_MLPERF_LOADGEN_SCENARIO: Offline MLC_MLPERF_RUN_STYLE: test