From 3a17bb98e7cbce5b0abc957c2869e99f53a59f99 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 04:03:52 +0530 Subject: [PATCH 1/4] Fixes for whisper, dont print python env at end --- script/app-mlperf-inference-mlcommons-python/meta.yaml | 3 +-- script/get-python3/meta.yaml | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 197d8ec94..551fe922c 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -1510,7 +1510,6 @@ variations: - tags: get,generic-python-lib,_package.setuptools-scm - tags: get,generic-python-lib,_package.librosa - tags: get,generic-python-lib,_package.transformers - version_max: "4.52.4" - tags: get,generic-python-lib,_package.openai-whisper # - tags: get,generic-python-lib,_package.evaluate # - tags: get,generic-python-lib,_package.absl-py @@ -1521,7 +1520,7 @@ variations: - "yes" names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_r2_downloader + - tags: get,dataset,whisper,_preprocessed names: - whisper-dataset skip_if_env: diff --git a/script/get-python3/meta.yaml b/script/get-python3/meta.yaml index b187346d2..434a00f0e 100644 --- a/script/get-python3/meta.yaml +++ b/script/get-python3/meta.yaml @@ -21,9 +21,6 @@ prehook_deps: inherit_variation_tags: true reuse_version: true tags: install,python,src -print_env_at_the_end: - MLC_PYTHON_BIN_WITH_PATH: Path to Python - MLC_PYTHON_VERSION: Python version tags: - get - python From fbd86f0ef63fbda8e4c5f70d8502eb498b11c271 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 04:47:11 +0530 Subject: [PATCH 2/4] adr -> ad in get-ml-model-whisper --- script/get-ml-model-whisper/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index bd7f4eaef..1003c365a 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -52,7 +52,7 @@ variations: _url.: - MLC_DOWNLOAD_URL rclone: - add_deps_recursive: + add_deps: dae: tags: _rclone env: @@ -72,9 +72,9 @@ variations: default: true group: download-tool r2_downloader: - add_deps_recursive: + add_deps: dae: tags: _r2_downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' - group: download-tool \ No newline at end of file + group: download-tool From b840cf26d78e688ded8b657aa408b06851c50a24 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 04:49:15 +0530 Subject: [PATCH 3/4] r2_downloader -> r2-downloader --- .../meta.yaml | 10 +++++----- script/app-mlperf-inference/meta.yaml | 12 ++++++------ script/download-and-extract/meta.yaml | 4 ++-- script/download-file/meta.yaml | 4 ++-- script/get-dataset-cnndm/meta.yaml | 18 +++++++++--------- script/get-dataset-whisper/meta.yaml | 10 +++++----- script/get-ml-model-llama3/meta.yaml | 12 ++++++------ script/get-ml-model-whisper/meta.yaml | 8 ++++---- .../meta.yaml | 12 ++++++------ 9 files changed, 45 insertions(+), 45 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 551fe922c..7980fa9f1 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -526,7 +526,7 @@ deps: - "yes" ## LLAMA3_1-8B - - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader + - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader names: - llama3-8b-model enable_if_env: @@ -1476,7 +1476,7 @@ variations: MLC_MODEL: llama3_1-8b deps: ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter skip_if_env: @@ -1490,7 +1490,7 @@ variations: MLC_MODEL: llama3_1-8b-edge deps: ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge skip_if_env: @@ -1514,7 +1514,7 @@ variations: # - tags: get,generic-python-lib,_package.evaluate # - tags: get,generic-python-lib,_package.absl-py # - tags: get,generic-python-lib,_package.rouge-score - - tags: get,ml-model,whisper,_r2_downloader,_mlc + - tags: get,ml-model,whisper,_r2-downloader,_mlc skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" @@ -1532,7 +1532,7 @@ variations: env: MLC_MODEL: deepseek-r1 deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2_downloader + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2-downloader names: - deepseek-r1-preprocessed-dataset skip_if_env: diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index bc6461a5a..cfcb5ef1a 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -969,7 +969,7 @@ variations: - "${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}" - "${{ MLC_DATASET_CNNDM_EVAL_PATH }}:${{ MLC_DATASET_CNNDM_EVAL_PATH }}" deps: - - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader + - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' @@ -1003,7 +1003,7 @@ variations: docker: deps: ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter enable_if_env: @@ -1038,7 +1038,7 @@ variations: docker: deps: ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge enable_if_env: @@ -1072,13 +1072,13 @@ variations: tags: run,accuracy,mlperf,_librispeech_whisper,_int64 docker: deps: - - tags: get,ml-model,whisper,_r2_downloader,_mlc + - tags: get,ml-model,whisper,_r2-downloader,_mlc enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_r2_downloader + - tags: get,dataset,whisper,_preprocessed,_mlc,_r2-downloader names: - whisper-dataset enable_if_env: @@ -1150,7 +1150,7 @@ variations: tags: run,accuracy,mlperf,_dataset_deepseek-r1 docker: deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2_downloader + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2-downloader enable_if_env: MLC_USE_DATASET_FROM_HOST: - 'yes' diff --git a/script/download-and-extract/meta.yaml b/script/download-and-extract/meta.yaml index 02aaf1175..8a3b17d07 100644 --- a/script/download-and-extract/meta.yaml +++ b/script/download-and-extract/meta.yaml @@ -125,9 +125,9 @@ variations: download-script: tags: _wget group: download-tool - r2_downloader: + r2-downloader: add_deps: download-script: - tags: _r2_downloader + tags: _r2-downloader group: download-tool versions: {} diff --git a/script/download-file/meta.yaml b/script/download-file/meta.yaml index e195b078f..aaf9bf1dd 100644 --- a/script/download-file/meta.yaml +++ b/script/download-file/meta.yaml @@ -77,9 +77,9 @@ variations: env: MLC_DOWNLOAD_TOOL: wget group: download-tool - r2_downloader: + r2-downloader: env: - MLC_DOWNLOAD_TOOL: r2_downloader + MLC_DOWNLOAD_TOOL: r2-downloader group: download-tool versions: {} diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index 4d38491f8..1bdf3db37 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -72,11 +72,11 @@ variations: add_deps_recursive: dae: tags: _rclone - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader dry-run: group: run-mode env: @@ -84,7 +84,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -135,7 +135,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH - validation,edge,llama3,mlc,r2_downloader: + validation,edge,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,edge @@ -153,7 +153,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH - validation,datacenter,llama3,mlc,r2_downloader: + validation,datacenter,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,datacenter @@ -171,7 +171,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH - calibation,llama3,mlc,r2_downloader: + calibation,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,calib @@ -185,7 +185,7 @@ tests: - variations_list: # - validation,edge,rclone,llama3,mlc,dry-run # - validation,datacenter,rclone,llama3,mlc,dry-run - - validation,edge,r2_downloader,llama3,mlc,dry-run - - validation,datacenter,r2_downloader,llama3,mlc,dry-run - - calibration,r2_downloader,llama3,mlc,dry-run + - validation,edge,r2-downloader,llama3,mlc,dry-run + - validation,datacenter,r2-downloader,llama3,mlc,dry-run + - calibration,r2-downloader,llama3,mlc,dry-run # - calibration,rclone,llama3,mlc,dry-run diff --git a/script/get-dataset-whisper/meta.yaml b/script/get-dataset-whisper/meta.yaml index 12fddf9e4..a436e6ef5 100644 --- a/script/get-dataset-whisper/meta.yaml +++ b/script/get-dataset-whisper/meta.yaml @@ -14,7 +14,7 @@ tests: run_inputs: - variations_list: - rclone,preprocessed,mlc,dry-run - - r2_downloader,preprocessed,mlc,dry-run + - r2-downloader,preprocessed,mlc,dry-run variations: preprocessed: group: dataset-type @@ -45,7 +45,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -59,10 +59,10 @@ variations: tags: _rclone default: true group: download-tool - r2_downloader: + r2-downloader: add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader group: download-tool rclone,preprocessed: env: @@ -95,6 +95,6 @@ variations: update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL - r2_downloader,preprocessed: + r2-downloader,preprocessed: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/whisper-dataset.uri diff --git a/script/get-ml-model-llama3/meta.yaml b/script/get-ml-model-llama3/meta.yaml index fd79fdc33..8b26d7fff 100644 --- a/script/get-ml-model-llama3/meta.yaml +++ b/script/get-ml-model-llama3/meta.yaml @@ -38,8 +38,8 @@ tests: run_inputs: - variations_list: - rclone,405b,mlc,dry-run - - r2_downloader,405b,mlc,dry-run - - r2_downloader,8b,mlc,dry-run + - r2-downloader,405b,mlc,dry-run + - r2-downloader,8b,mlc,dry-run variations: fp16: default: true @@ -86,7 +86,7 @@ variations: adr: dae: extra_cache_tags: llama3,dataset,rclone - mlc,r2_downloader: + mlc,r2-downloader: env: MLC_DOWNLOAD_URL: https://llama3-1.mlcommons-storage.org/metadata/<<>>.uri adr: @@ -108,11 +108,11 @@ variations: MLC_TMP_REQUIRE_DOWNLOAD: - yes default: true - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader dry-run: group: run-mode env: @@ -120,7 +120,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x hf: diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index 1003c365a..ece8ec1fc 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -16,7 +16,7 @@ tests: run_inputs: - variations_list: - rclone,mlc,dry-run - - r2_downloader,mlc,dry-run + - r2-downloader,mlc,dry-run uid: 3bea2356e97f47b1 variations: dry-run: @@ -26,7 +26,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -71,10 +71,10 @@ variations: tags: get,rclone-config,_mlc-inference default: true group: download-tool - r2_downloader: + r2-downloader: add_deps: dae: - tags: _r2_downloader + tags: _r2-downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' group: download-tool diff --git a/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml b/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml index 8c0da0989..0cc5f359c 100644 --- a/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml +++ b/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml @@ -28,18 +28,18 @@ variations: validation,rclone: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/deepseek_r1/mlperf_deepseek_r1_dataset_4388_fp8_eval.pkl - validation,r2_downloader: + validation,r2-downloader: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-dataset-4388-fp8-eval.uri calibration: group: dataset-type env: MLC_PREPROCESSED_DATASET_TYPE: calibration - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader rclone: group: download-tool add_deps_recursive: @@ -66,7 +66,7 @@ variations: calibration,rclone: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/deepseek_r1/mlperf_deepseek_r1_calibration_dataset_500_fp8_eval.pkl - calibration,r2_downloader: + calibration,r2-downloader: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-calibration-dataset-500-fp8-eval.uri mlc: @@ -94,8 +94,8 @@ variations: tests: run_inputs: - variations_list: - - calibration,r2_downloader,mlc,dry-run - - validation,r2_downloader,mlc,dry-run + - calibration,r2-downloader,mlc,dry-run + - validation,r2-downloader,mlc,dry-run # - validation,rclone,mlc,dry-run # - calibration,rclone,mlc,dry-run From 39fa33dbb7e1b5dc731f28fcb447960eaed773e6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 04:56:12 +0530 Subject: [PATCH 4/4] adr fix in get-ml-model-whisper --- script/get-ml-model-whisper/meta.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index ece8ec1fc..de84c768b 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -46,14 +46,14 @@ variations: force_env_keys: - MLC_OUTDIRNAME names: - - dae + - whisper-model-dae tags: download-and-extract update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL rclone: - add_deps: - dae: + adr: + whisper-model-dae: tags: _rclone env: MLC_DOWNLOAD_URL: 'mlc-inference:mlcommons-inference-wg-public/Whisper/model/' @@ -72,8 +72,8 @@ variations: default: true group: download-tool r2-downloader: - add_deps: - dae: + adr: + whisper-model-dae: tags: _r2-downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri'