From 282de38ee09c55e6a29e13969faf69bd68a5da48 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Jul 2025 00:20:33 +0100 Subject: [PATCH 001/124] Update meta.yaml --- script/generate-mlperf-inference-submission/meta.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/generate-mlperf-inference-submission/meta.yaml b/script/generate-mlperf-inference-submission/meta.yaml index 4060c6058..4125b7eb7 100644 --- a/script/generate-mlperf-inference-submission/meta.yaml +++ b/script/generate-mlperf-inference-submission/meta.yaml @@ -124,6 +124,8 @@ variations: tags: _wg-inference mlperf-inference-submission-checker: tags: _wg-inference + default_env: + MLC_MLPERF_SUBMISSION_CHECKER_VERSION: v5.1 deps: - names: - inference-src @@ -140,6 +142,8 @@ variations: tags: _wg-automotive mlperf-inference-submission-checker: tags: _wg-automotive + default_env: + MLC_MLPERF_SUBMISSION_CHECKER_VERSION: v0.5 deps: - names: - automotive-src From 4ac3dd30cd3ae8e37463b8a1f81070a1babcd465 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 15 Jul 2025 04:40:49 +0000 Subject: [PATCH 002/124] [Automated Commit] Document script/generate-mlperf-inference-submission/meta.yaml [skip ci] --- script/generate-mlperf-inference-submission/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/script/generate-mlperf-inference-submission/README.md b/script/generate-mlperf-inference-submission/README.md index 9e66372e8..416a01828 100644 --- a/script/generate-mlperf-inference-submission/README.md +++ b/script/generate-mlperf-inference-submission/README.md @@ -41,6 +41,7 @@ mlcr generate,submission,mlperf,mlperf-inference,inference,mlcommons,inference-s | Name | Description | Choices | Default | |------|-------------|---------|------| | `--analyzer_settings_file` | | | `` | +| `--status` | | | `` | | `--category` | | | `` | | `--clean` | | | `` | | `--dashboard` | | | `` | From 3446a226d10d0b18d4d13f6fe76a73dd71ead38d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Jul 2025 10:48:57 +0530 Subject: [PATCH 003/124] Update pull_request_template.md --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8db080676..0c9be0346 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,5 +1,6 @@ ### ๐Ÿงพ PR Checklist - [ ] Target branch is `dev` +- [ ] Changes have been tested locally ๐Ÿ“Œ Note: PRs must be raised against `dev`. Do not commit directly to `main`. From a9b50c5e9e88965307dd98ac5d56df3c71d566e6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 15 Jul 2025 17:46:19 +0530 Subject: [PATCH 004/124] Changes for supporting lint action (#509) --- automation/script/lint.py | 160 ++++++++++++++++++++++++++++++++++++ automation/script/module.py | 6 ++ 2 files changed, 166 insertions(+) create mode 100644 automation/script/lint.py diff --git a/automation/script/lint.py b/automation/script/lint.py new file mode 100644 index 000000000..63825c22f --- /dev/null +++ b/automation/script/lint.py @@ -0,0 +1,160 @@ +import os +import yaml +import copy +from mlc import utils +from utils import * + + +def lint_meta(self_module, input_params): + """ + Lints MLC script metadata files by fixing key order and validating structure. + + Args: + self_module: Reference to the current module for internal calls. + i: Dictionary containing input parameters. + + Returns: + Dictionary with the result of the operation. Keys: + - 'return': 0 on success, >0 on error. + - 'error': Error message (if any). + """ + + # Extract and handle basic inputs + quiet = input_params.get('quiet', False) + logger = self_module.logger + env = input_params.get('env', {}) + generic_inputs = self_module.input_flags_converted_to_env + generic_inputs = dict(sorted(generic_inputs.items())) + + # Search for scripts + search_result = self_module.search(input_params.copy()) + if search_result['return'] > 0: + return search_result + + scripts_list = search_result['list'] + if not scripts_list: + return {'return': 1, 'error': 'No scripts were found'} + + env = input_params.get('env', {}) + state_data = input_params.get('state', {}) + constant_vars = input_params.get('const', {}) + constant_state = input_params.get('const_state', {}) + tag_values = input_params.get('tags', '').split(",") + variation_tags = [tag[1:] for tag in tag_values if tag.startswith("_")] + + # Iterate over scripts + for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): + metadata = script.meta + script_directory = script.path + script_tags = metadata.get("tags", []) + script_alias = metadata.get('alias', '') + script_uid = metadata.get('uid', '') + script_input_mapping = metadata.get('input_mapping', {}) + script_input_description = metadata.get('input_description', {}) + script_repo = script.repo + + # Sort YAML keys + sort_result = sort_meta_yaml_file(script_directory, quiet) + if sort_result['return'] > 0: + if not quiet: + logger.error( + f"Failed to sort YAML keys for {script_alias}: {sort_result.get('error', '')}") + elif sort_result.get('modified', False): + if not quiet: + logger.info(f"Sorted YAML keys for {script_alias}") + elif not sort_result.get('modified', False): + if not quiet: + logger.info( + f"No input mapping or variations keys to be sorted for {script_alias}") + + return {'return': 0} + + +def sort_meta_yaml_file(script_directory, quiet=False): + """ + Sort specific keys in the meta.yaml file and save it back to disk. + + Args: + script_directory: Path to the script directory + quiet: Whether to suppress output messages + + Returns: + Dictionary with 'return' (0 on success, >0 on error), 'modified' (bool), + and 'error' (if any) + """ + + try: + # Find meta.yaml file + meta_yaml_path = None + for filename in ['meta.yaml', 'meta.yml']: + potential_path = os.path.join(script_directory, filename) + if os.path.exists(potential_path): + meta_yaml_path = potential_path + break + + if not meta_yaml_path: + return {'return': 1, 'error': 'meta.yaml file not found', + 'modified': False} + + # Read current YAML content + with open(meta_yaml_path, 'r', encoding='utf-8') as file: + data = yaml.safe_load(file) + + if not isinstance(data, dict): + return { + 'return': 1, 'error': 'YAML does not contain a dictionary', 'modified': False} + + # Store original for comparison + original_data = copy.deepcopy(data) + + # Sort input_mapping alphabetically + if 'input_mapping' in data and isinstance(data['input_mapping'], dict): + data['input_mapping'] = dict(sorted(data['input_mapping'].items())) + + # Sort variations: with 'group' first, then without 'group' + if 'variations' in data and isinstance(data['variations'], dict): + variations = data['variations'] + + # Separate variations with and without 'group' key + with_group = [] + without_group = [] + + for key, value in variations.items(): + if isinstance(value, dict) and 'group' in value: + with_group.append((key, value)) + else: + without_group.append((key, value)) + + # Sort both lists alphabetically by key + with_group.sort(key=lambda x: x[0]) + without_group.sort(key=lambda x: x[0]) + + # Combine them: with_group first, then without_group + sorted_variations = {} + for key, value in with_group + without_group: + sorted_variations[key] = value + + data['variations'] = sorted_variations + + # Check if anything changed (including order) + original_yaml = yaml.dump( + original_data, + default_flow_style=False, + sort_keys=False) + new_yaml = yaml.dump(data, default_flow_style=False, sort_keys=False) + + if original_yaml == new_yaml: + return {'return': 0, 'modified': False} + + # Write the sorted YAML back to file + with open(meta_yaml_path, 'w', encoding='utf-8') as file: + yaml.dump(data, file, default_flow_style=False, sort_keys=False, + allow_unicode=True, width=1000, indent=2) + + if not quiet: + print(f"Sorted YAML keys in {meta_yaml_path}") + + return {'return': 0, 'modified': True, 'sorted_data': data} + + except Exception as e: + return {'return': 1, 'error': str(e), 'modified': False} diff --git a/automation/script/module.py b/automation/script/module.py index b1759c1c7..9c800322a 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4467,6 +4467,12 @@ def doc(self, i): from script.doc import generate_doc return generate_doc(self, i) + + ############################################################ + + def lint(self, i): + from script.lint import lint_meta + return lint_meta(self, i) ############################################################ From a147e8ecf9da697d3476c2231c0c1511fc9da9ed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 15 Jul 2025 12:16:33 +0000 Subject: [PATCH 005/124] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index 9c800322a..9d793823b 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4467,7 +4467,7 @@ def doc(self, i): from script.doc import generate_doc return generate_doc(self, i) - + ############################################################ def lint(self, i): From b9dbbf32f3e37f33e741f7b6b02dd8be833eaccc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Jul 2025 13:24:25 +0100 Subject: [PATCH 006/124] Update mkdocs.yml --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index ebd48a20a..c9c12d71d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,6 @@ site_name: MLPerf Automation Documentation repo_url: https://github.com/mlcommons/mlperf-automations +site_url: https://docs.mlcommons.org/mlperf-automations theme: name: material logo: img/logo_v2.svg From 9883958b83eefeaeb1b854de7e7c871d34cdf0aa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 15 Jul 2025 20:58:07 +0100 Subject: [PATCH 007/124] Support simultaneous activation of multiple dynamic variations (#512) --- automation/script/module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/automation/script/module.py b/automation/script/module.py index 9d793823b..f2b37c81e 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2461,6 +2461,7 @@ def _update_state_from_variations(self, i, meta, variation_tags, variations, env variation_meta = variations[variation_tag] if variation_tag_dynamic_suffix: + variation_meta = copy.deepcopy(variation_meta) self._update_variation_meta_with_dynamic_suffix( variation_meta, variation_tag_dynamic_suffix) From 3f48d6bd246babc97be2b6a31a5aa1a48fec4008 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 16 Jul 2025 18:17:05 +0100 Subject: [PATCH 008/124] Support base variations inside combination of variations, fixes #38, support list in update_tags_from_env (#513) * Support base variations inside combination of variations, fixes #38 * Added patch.# variation for get-git-repo * Support lists in update_tags_from_env_with_prefix --- automation/script/module.py | 135 +++++++++++++++++++++++--------- script/get-git-repo/meta.yaml | 5 ++ script/set-cpu-frequency/run.sh | 3 + 3 files changed, 105 insertions(+), 38 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index f2b37c81e..954e0c444 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2574,7 +2574,63 @@ def _update_state_from_variations(self, i, meta, variation_tags, variations, env return {'return': 0, 'variation_tags_string': variation_tags_string, 'explicit_variation_tags': explicit_variation_tags, 'warnings': warnings} + def _add_base_variations( + self, + variation_name, + variations, + variation_tags, + tmp_variations, + excluded_variation_tags + ): + """ + Adds base variations for a given variation_name into variation_tags + and updates tmp_variations if valid. + """ + + if "base" not in variations[variation_name]: + return {'return': 0} + + for base_variation in variations[variation_name]["base"]: + tag_to_append = None + dynamic_base_variation = False + dynamic_base_variation_already_added = False + + # Handle dynamic variation + if base_variation not in variations: + base_variation_dynamic = self._get_name_for_dynamic_variation_tag( + base_variation) + if not base_variation_dynamic or base_variation_dynamic not in variations: + return { + 'return': 1, + 'error': f'Variation "{base_variation}" specified as base variation of "{variation_name}" is not existing' + } + dynamic_base_variation = True + base_prefix = base_variation_dynamic.split(".")[0] + "." + + # We allow repeated dynamic variations like _patch.1,_patch.2,_patch.3 + # for tag in variation_tags: + # if tag.startswith(base_prefix): + # dynamic_base_variation_already_added = True + # break + + # Append if not already added + if base_variation not in variation_tags and not dynamic_base_variation_already_added: + tag_to_append = base_variation + + # Validate exclusion list + if tag_to_append: + if tag_to_append in excluded_variation_tags: + return { + 'return': 1, + 'error': f'Variation "{tag_to_append}" specified as base variation for the variation is in the excluded list "{variation_name}"' + } + variation_tags.append(tag_to_append) + tmp_variations[tag_to_append] = False + + return {'return': 0} + ########################################################################## + def _update_variation_tags_from_variations( self, variation_tags, variations, variation_groups, excluded_variation_tags): @@ -2612,40 +2668,15 @@ def _update_variation_tags_from_variations( variation_name = self._get_name_for_dynamic_variation_tag( variation_name) - # TODO: Move this to a function and apply it for combination of variations too - # base variations are automatically turned on. Only - # variations outside of any variation group can be added as - # a base_variation - if "base" in variations[variation_name]: - base_variations = variations[variation_name]["base"] - for base_variation in base_variations: - dynamic_base_variation = False - dynamic_base_variation_already_added = False - if base_variation not in variations: - base_variation_dynamic = self._get_name_for_dynamic_variation_tag( - base_variation) - if not base_variation_dynamic or base_variation_dynamic not in variations: - return {'return': 1, 'error': 'Variation "{}" specified as base variation of "{}" is not existing'.format( - base_variation, variation_name)} - else: - dynamic_base_variation = True - base_prefix = base_variation_dynamic.split(".")[ - 0] + "." - for x in variation_tags: - if x.startswith(base_prefix): - dynamic_base_variation_already_added = True - - if base_variation not in variation_tags and not dynamic_base_variation_already_added: - tag_to_append = base_variation - - if tag_to_append: - if tag_to_append in excluded_variation_tags: - return {'return': 1, 'error': 'Variation "{}" specified as base variation for the variation is in the excluded list "{}" '.format( - tag_to_append, variation_name)} - variation_tags.append(tag_to_append) - tmp_variations[tag_to_append] = False - - tag_to_append = None + result = self._add_base_variations( + variation_name, + variations, + variation_tags, + tmp_variations, + excluded_variation_tags + ) + if result.get('return', 0) > 0: + return result # default_variations dictionary specifies the # default_variation for each variation group. A default @@ -2677,6 +2708,16 @@ def _update_variation_tags_from_variations( combined_variation_meta = variations[combined_variation] tmp_combined_variations[combined_variation] = True + result = self._add_base_variations( + combined_variation, + variations, + variation_tags, + tmp_combined_variations, + excluded_variation_tags + ) + if result.get('return', 0) > 0: + return result + r = self._get_variation_tags_from_default_variations( combined_variation_meta, variations, @@ -3364,9 +3405,17 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a for t in update_tags_from_env_with_prefix: for key in update_tags_from_env_with_prefix[t]: if str(d.get('env', {}).get(key, '')).strip() != '': - d['tags'] += "," + t + str(d.get('env')[key]) + if isinstance(d.get('env')[key], str): + d['tags'] += "," + t + str(d.get('env')[key]) + elif isinstance(d.get('env')[key], list): + for item in d.get('env')[key]: + d['tags'] += "," + t + str(item) elif str(env.get(key, '')).strip() != '': - d['tags'] += "," + t + str(env[key]) + if isinstance(env[key], str): + d['tags'] += "," + t + str(env[key]) + elif isinstance(env[key], list): + for item in env[key]: + d['tags'] += "," + t + str(item) for key in clean_env_keys_deps: if '?' in key or '*' in key: @@ -3405,8 +3454,18 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a update_tags_from_env = d.get("update_tags_from_env", []) for t in update_tags_from_env: - if env.get(t, '').strip() != '': - d['tags'] += "," + env[t] + if str(d.get('env', {}).get(t, '')).strip() != '': + if isinstance(d.get('env')[t], str): + d['tags'] += "," + str(d.get('env')[t]) + elif isinstance(d.get('env')[t], list): + for item in d.get('env')[t]: + d['tags'] += "," + str(item) + elif str(env.get(t, '')).strip() != '': + if isinstance(env[t], str): + d['tags'] += "," + str(env[t]) + elif isinstance(env[t], list): + for item in env[t]: + d['tags'] += "," + str(item) update_tags_if_env = d.get("update_tags_if_env", []) for t in update_tags_if_env: diff --git a/script/get-git-repo/meta.yaml b/script/get-git-repo/meta.yaml index 468468bb5..e52110035 100644 --- a/script/get-git-repo/meta.yaml +++ b/script/get-git-repo/meta.yaml @@ -67,6 +67,11 @@ variations: patch: env: MLC_GIT_PATCH: 'yes' + patch.#: + env: + MLC_GIT_PATCH: 'yes' + +,MLC_GIT_PATCH_FILEPATHS: + - '#' pr-to-apply.#: env: MLC_GIT_PR_TO_APPLY: '#' diff --git a/script/set-cpu-frequency/run.sh b/script/set-cpu-frequency/run.sh index 57b22872a..1bf863667 100644 --- a/script/set-cpu-frequency/run.sh +++ b/script/set-cpu-frequency/run.sh @@ -24,6 +24,9 @@ case "$DRIVER_KEY" in echo "โ†’ intel_pstate: disabling turbo, setting performance governor" echo 0 | ${MLC_SUDO} tee /sys/devices/system/cpu/intel_pstate/no_turbo >/dev/null ${MLC_SUDO} cpupower frequency-set -g performance + echo "" + echo "Note: intel_pstate does _not_ support a userspace/fixed frequency mode." + echo "If you need a precise kHz, switch back to acpi-cpufreq in your kernel cmdline." ;; amd-pstate) From 8e393f8b97e1887accfc3eb2efa31c3bcca41223 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 16 Jul 2025 17:17:29 +0000 Subject: [PATCH 009/124] [Automated Commit] Document script/get-git-repo/meta.yaml [skip ci] --- script/get-git-repo/README.md | 97 +++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 script/get-git-repo/README.md diff --git a/script/get-git-repo/README.md b/script/get-git-repo/README.md new file mode 100644 index 000000000..9c68729c1 --- /dev/null +++ b/script/get-git-repo/README.md @@ -0,0 +1,97 @@ +# README for get-git-repo +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +``` +mkdir /mnt/user/MLC +ln -s /mnt/user/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,git,repo,repository,clone +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--branch` | | | `` | +| `--depth` | | | `--depth 4` | +| `--env_key` | | | `` | +| `--folder` | | | `repo` | +| `--patch` | | | `no` | +| `--pull` | | | `` | +| `--submodules` | | | ` --recurse-submodules` | +| `--update` | Alias for pull | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Checkout + +- `branch.#` _(# can be substituted dynamically)_ +- `tag.#` _(# can be substituted dynamically)_ + +### Git-history + +- `full-history` +- `short-history` (default) + +### Post-checkout + +- `sha.#` _(# can be substituted dynamically)_ + +### Repo + +- `repo.#` _(# can be substituted dynamically)_ + +### Ungrouped + +- `cherrypicks.#` _(# can be substituted dynamically)_ +- `lfs` +- `no-recurse-submodules` +- `patch` +- `patch.#` _(# can be substituted dynamically)_ +- `pr-to-apply.#` _(# can be substituted dynamically)_ +- `submodules.#` _(# can be substituted dynamically)_ From d99ef6d2c712cdc1872640bb95635cd3e5472d37 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 17 Jul 2025 22:01:37 +0100 Subject: [PATCH 010/124] Fix gcc installed path (#518) * Fix gcc installed path --- script/get-gcc/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-gcc/customize.py b/script/get-gcc/customize.py index 25e39a781..9c21b8639 100644 --- a/script/get-gcc/customize.py +++ b/script/get-gcc/customize.py @@ -85,7 +85,8 @@ def postprocess(i): found_path = os.path.dirname(found_file_path) - env['MLC_GCC_INSTALLED_PATH'] = found_path + env['MLC_GCC_INSTALLED_PATH'] = os.path.dirname( + found_path) # /usr in case of /usr/bin/gcc file_name_c = os.path.basename(found_file_path) # G: changed next line to handle cases like gcc-8 From 7fa2e0b3ebfdd8e3767f280092c0819c9e45fa68 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Jul 2025 16:53:56 +0530 Subject: [PATCH 011/124] Fix aocc version check regular expression --- script/get-aocc/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index d5890e103..e76f8123b 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -73,7 +73,7 @@ def preprocess(i): def detect_version(i): logger = i['automation'].logger - r = i['automation'].parse_version({'match_text': r'CLANG:\sAOCC_([\d.]+-Build#[\d]+)', + r = i['automation'].parse_version({'match_text': r'CLANG:\sAOCC_([\d.]+(?:-[\w]+)?-Build#[\d]+)', 'group_number': 1, 'env_key': 'MLC_AOCC_VERSION', 'which_env': i['env']}) From 7c9ff7635c927a6045f71935a929356785f23a4f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 18 Jul 2025 12:26:44 +0100 Subject: [PATCH 012/124] Fix aocc version check regular expression (#521) --- script/get-aocc/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-aocc/customize.py b/script/get-aocc/customize.py index d5890e103..e76f8123b 100644 --- a/script/get-aocc/customize.py +++ b/script/get-aocc/customize.py @@ -73,7 +73,7 @@ def preprocess(i): def detect_version(i): logger = i['automation'].logger - r = i['automation'].parse_version({'match_text': r'CLANG:\sAOCC_([\d.]+-Build#[\d]+)', + r = i['automation'].parse_version({'match_text': r'CLANG:\sAOCC_([\d.]+(?:-[\w]+)?-Build#[\d]+)', 'group_number': 1, 'env_key': 'MLC_AOCC_VERSION', 'which_env': i['env']}) From 6438e1d61abfcff6d1b02a6f07fd2f93df9ac9f9 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 18 Jul 2025 17:00:16 +0530 Subject: [PATCH 013/124] Return generated readme paths (#515) --- automation/script/doc.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/automation/script/doc.py b/automation/script/doc.py index 7d69685a0..0cee29d63 100644 --- a/automation/script/doc.py +++ b/automation/script/doc.py @@ -43,7 +43,10 @@ def generate_doc(self_module, input_params): tag_values = input_params.get('tags', '').split(",") variation_tags = [tag[1:] for tag in tag_values if tag.startswith("_")] - # Step 4: Iterate over scripts and generate Dockerfile + # capture path of generated README's for each script + generated_readme_paths = [] + + # Step 4: Iterate over scripts and generate readme for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): metadata = script.meta script_directory = script.path @@ -60,8 +63,10 @@ def generate_doc(self_module, input_params): generic_inputs) if r['return'] > 0: continue + else: + generated_readme_paths.append(os.path.join(script_directory, "README.md")) - return {'return': 0} + return {'return': 0, 'generated_readme_paths': generated_readme_paths} def get_setup_readme(script_repo): From f8e55da61802aac1b8c2c68a4aa24930f61de98e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 18 Jul 2025 11:30:30 +0000 Subject: [PATCH 014/124] [Automated Commit] Format Codebase [skip ci] --- automation/script/doc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automation/script/doc.py b/automation/script/doc.py index 0cee29d63..7c8fb6538 100644 --- a/automation/script/doc.py +++ b/automation/script/doc.py @@ -64,7 +64,8 @@ def generate_doc(self_module, input_params): if r['return'] > 0: continue else: - generated_readme_paths.append(os.path.join(script_directory, "README.md")) + generated_readme_paths.append( + os.path.join(script_directory, "README.md")) return {'return': 0, 'generated_readme_paths': generated_readme_paths} From 4a1c06f368ea50f53431892cdda2bfdb0106f002 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 18 Jul 2025 17:01:14 +0530 Subject: [PATCH 015/124] fix for issue 479 (#516) --- script/build-dockerfile/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index b240e4164..16685c7a4 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -293,7 +293,8 @@ def preprocess(i): dockerfile_env_input_string = "" for docker_env_key in dockerfile_env: dockerfile_env_input_string = dockerfile_env_input_string + " --env." + \ - docker_env_key + "=" + str(dockerfile_env[docker_env_key]) + docker_env_key + "=" + \ + str(dockerfile_env[docker_env_key]).replace("\n", "\\n") workdir = env.get('WORKDIR', '') if workdir == '': From 976285f6e386479ec616ab800f5538c601b85de9 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 18 Jul 2025 17:03:20 +0530 Subject: [PATCH 016/124] Update PR checklist (#514) * Update PR checklist * Update pull_request_template.md --- .github/pull_request_template.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 0c9be0346..3e70589a3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,23 @@ ### ๐Ÿงพ PR Checklist - [ ] Target branch is `dev` -- [ ] Changes have been tested locally ๐Ÿ“Œ Note: PRs must be raised against `dev`. Do not commit directly to `main`. + +### โœ… Testing & CI +- [ ] No existing GitHub Actions are failing because of this change + +### ๐Ÿ“ File Hygiene & Output Handling +- [ ] No unintended files (e.g., logs, cache, temp files, __pycache__, output folders) are committed + +### ๐Ÿ“ Comments & Communication +- [ ] Proper inline comments are added to explain important or non-obvious changes +- [ ] PR title and description clearly state what the PR does and why +- [ ] Related issues (if any) are properly referenced (`Fixes #`, `Related to #`, etc.) +- [ ] All reviewer feedback has been addressed + +### ๐Ÿ›ก๏ธ Safety & Security +- [ ] No secrets or credentials are committed +- [ ] Paths, shell commands, and environment handling are safe and portable + + From 0b20b8c9590fbcb6c9cd19c8ecf96ab49b5708e8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 19 Jul 2025 07:12:00 +0530 Subject: [PATCH 017/124] Handle int,float in update_tags_from_env --- automation/script/module.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 954e0c444..e86a6d646 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3405,13 +3405,13 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a for t in update_tags_from_env_with_prefix: for key in update_tags_from_env_with_prefix[t]: if str(d.get('env', {}).get(key, '')).strip() != '': - if isinstance(d.get('env')[key], str): + if isinstance(d.get('env')[key], (str, int, float)): d['tags'] += "," + t + str(d.get('env')[key]) elif isinstance(d.get('env')[key], list): for item in d.get('env')[key]: d['tags'] += "," + t + str(item) elif str(env.get(key, '')).strip() != '': - if isinstance(env[key], str): + if isinstance(env[key], (str, int, float)): d['tags'] += "," + t + str(env[key]) elif isinstance(env[key], list): for item in env[key]: @@ -3455,13 +3455,13 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a update_tags_from_env = d.get("update_tags_from_env", []) for t in update_tags_from_env: if str(d.get('env', {}).get(t, '')).strip() != '': - if isinstance(d.get('env')[t], str): + if isinstance(d.get('env')[t], (str, int, float)): d['tags'] += "," + str(d.get('env')[t]) elif isinstance(d.get('env')[t], list): for item in d.get('env')[t]: d['tags'] += "," + str(item) elif str(env.get(t, '')).strip() != '': - if isinstance(env[t], str): + if isinstance(env[t], (str, int, float)): d['tags'] += "," + str(env[t]) elif isinstance(env[t], list): for item in env[t]: From 174cd6f2db8fc15efae40818f283773e55b5e089 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 19 Jul 2025 01:42:18 +0000 Subject: [PATCH 018/124] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index e86a6d646..6c15f76c5 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3405,7 +3405,8 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a for t in update_tags_from_env_with_prefix: for key in update_tags_from_env_with_prefix[t]: if str(d.get('env', {}).get(key, '')).strip() != '': - if isinstance(d.get('env')[key], (str, int, float)): + if isinstance( + d.get('env')[key], (str, int, float)): d['tags'] += "," + t + str(d.get('env')[key]) elif isinstance(d.get('env')[key], list): for item in d.get('env')[key]: From 4fc6e53863d0c7ac4512aecdcc62356d068fc700 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 21 Jul 2025 11:29:40 +0100 Subject: [PATCH 019/124] Update document-scripts.yml | Use venv for document script --- .github/workflows/document-scripts.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/document-scripts.yml b/.github/workflows/document-scripts.yml index 6984ff719..9298efbf9 100644 --- a/.github/workflows/document-scripts.yml +++ b/.github/workflows/document-scripts.yml @@ -65,7 +65,8 @@ jobs: - name: Document meta.yaml file run: | echo "Documenting ${{ matrix.modified_metas.file }}" - + python3 -m venv mlcflow + . mlcflow/bin/activate pip install mlcflow mlc add repo automation-scripts mlc doc script ${{ matrix.modified_metas.uid}} --quiet From 522ca999af25f441b54de09f21aa00fed823d89a Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 21 Jul 2025 20:42:54 +0530 Subject: [PATCH 020/124] Fix backup url env --- script/get-ml-model-resnet50/meta.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/get-ml-model-resnet50/meta.yaml b/script/get-ml-model-resnet50/meta.yaml index a752747ff..24178cf97 100644 --- a/script/get-ml-model-resnet50/meta.yaml +++ b/script/get-ml-model-resnet50/meta.yaml @@ -30,6 +30,7 @@ prehook_deps: update_tags_from_env_with_prefix: _url.: - MLC_PACKAGE_URL + - MLC_PACKAGE_URL1 print_env_at_the_end: MLC_ML_MODEL_FILE_WITH_PATH: Path to the ML model tags: @@ -132,12 +133,12 @@ variations: env: MLC_DOWNLOAD_CHECKSUM: f6a4da60cd5f084d97efc2c1ddb10beb MLC_PACKAGE_URL: https://armi.in/files/resnet50_v1_op11/resnet50_v1.onnx - MLC_PACKAGE_URL1: https://zenodo.org/record/4735647/files/resnet50_v1.onnx + MLC_DOWNLOAD_URL1: https://zenodo.org/record/4735647/files/resnet50_v1.onnx onnx,opset-8: env: MLC_DOWNLOAD_CHECKSUM: a638cf028b5870da29e09ccc2f7182e7 MLC_PACKAGE_URL: https://armi.in/files/resnet50_v1.onnx - MLC_PACKAGE_URL1: https://zenodo.org/record/2592612/files/resnet50_v1.onnx + MLC_DOWNLOAD_URL1: https://zenodo.org/record/2592612/files/resnet50_v1.onnx onnxruntime: alias: onnx opset-11: From 322526f912ea34a7883d2557285a8dadcf96775e Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 21 Jul 2025 20:44:06 +0530 Subject: [PATCH 021/124] clean code --- script/get-ml-model-resnet50/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/script/get-ml-model-resnet50/meta.yaml b/script/get-ml-model-resnet50/meta.yaml index 24178cf97..69ab60401 100644 --- a/script/get-ml-model-resnet50/meta.yaml +++ b/script/get-ml-model-resnet50/meta.yaml @@ -30,7 +30,6 @@ prehook_deps: update_tags_from_env_with_prefix: _url.: - MLC_PACKAGE_URL - - MLC_PACKAGE_URL1 print_env_at_the_end: MLC_ML_MODEL_FILE_WITH_PATH: Path to the ML model tags: From 28aa5aff1a396def1ea91b3d262e24109a262ac9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Jul 2025 15:24:14 +0000 Subject: [PATCH 022/124] [Automated Commit] Document script/get-ml-model-resnet50/meta.yaml [skip ci] --- script/get-ml-model-resnet50/README.md | 90 ++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 script/get-ml-model-resnet50/README.md diff --git a/script/get-ml-model-resnet50/README.md b/script/get-ml-model-resnet50/README.md new file mode 100644 index 000000000..d6e3dd61f --- /dev/null +++ b/script/get-ml-model-resnet50/README.md @@ -0,0 +1,90 @@ +# README for get-ml-model-resnet50 +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +``` +mkdir /mnt/user/MLC +ln -s /mnt/user/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,raw,ml-model,resnet50,ml-model-resnet50,image-classification +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Framework + +- `ncnn` +- `onnx` (alias: onnxruntime) (default) +- `pytorch` +- `tensorflow` (alias: tf) +- `tflite` + +### Model-output + +- `argmax` (default) +- `no-argmax` + +### Opset-version + +- `opset-11` +- `opset-8` + +### Precision + +- `fp32` (default) +- `int8` +- `uint8` + +### Ungrouped + +- `batch_size.#` _(# can be substituted dynamically)_ +- `batch_size.1` +- `fix-input-shape` +- `from-tf` +- `huggingface_default` From 6a523e3aa6ca08eef566451562ea0a3950e73e04 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:56:13 +0530 Subject: [PATCH 023/124] Install ca-certificates on launching container with MLC --- script/build-dockerfile/dockerinfo.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-dockerfile/dockerinfo.json b/script/build-dockerfile/dockerinfo.json index 2b222c01c..f76510841 100644 --- a/script/build-dockerfile/dockerinfo.json +++ b/script/build-dockerfile/dockerinfo.json @@ -28,7 +28,7 @@ "package-manager-update-cmd": "apt-get update -y", "package-manager-get-cmd": "apt-get install -y", "packages": [ - "python3", "python3-pip", "git", "sudo", "wget", "python3-venv", "systemctl", "unzip" + "python3", "python3-pip", "git", "sudo", "wget", "python3-venv", "systemctl", "unzip", "ca-certificates" ], "versions": { "18.04": { From 7847589f2e82f9323a64ec687d81fc4685b25195 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 23 Jul 2025 12:56:10 +0530 Subject: [PATCH 024/124] Migrate the dataset and model downloads to R2 --- script/app-mlperf-inference/meta.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index f00790c92..bbcc99fa8 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -1001,7 +1001,7 @@ variations: - "${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}" - "${{ MLC_DATASET_CNNDM_EVAL_PATH }}:${{ MLC_DATASET_CNNDM_EVAL_PATH }}" deps: - - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct + - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' @@ -1009,7 +1009,7 @@ variations: - llama3_1-8b - llama3-8b ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_rclone + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader names: - cnndm-llama3-edge enable_if_env: @@ -1018,7 +1018,7 @@ variations: MLC_USE_DATASET_FROM_HOST: - 'yes' ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_rclone + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader names: - cnndm-llama3-datacenter enable_if_env: @@ -1052,13 +1052,13 @@ variations: tags: run,accuracy,mlperf,_librispeech_whisper,_int64 docker: deps: - - tags: get,ml-model,whisper,_rclone,_mlc + - tags: get,ml-model,whisper,_r2_downloader,_mlc enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_rclone + - tags: get,dataset,whisper,_preprocessed,_mlc,_r2_downloader names: - whisper-dataset enable_if_env: @@ -1130,7 +1130,7 @@ variations: tags: run,accuracy,mlperf,_dataset_deepseek-r1 docker: deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_rclone + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2_downloader enable_if_env: MLC_USE_DATASET_FROM_HOST: - 'yes' From 5a76d192d98d314a8a4aa3e599e193833beab9b2 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 23 Jul 2025 13:04:42 +0530 Subject: [PATCH 025/124] migrate dataset and model downloads to r2 --- .../app-mlperf-inference-mlcommons-python/meta.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 68980d98d..2ea27873b 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -524,7 +524,7 @@ deps: - "yes" ## LLAMA3_1-8B - - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct + - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader names: - llama3-8b-model enable_if_env: @@ -1468,7 +1468,7 @@ variations: - llama3_1-8b - llama3-8b ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_rclone + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader names: - cnndm-llama3-edge enable_if_env: @@ -1478,7 +1478,7 @@ variations: MLC_RUN_STATE_DOCKER: - "yes" ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_rclone + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader names: - cnndm-llama3-datacenter enable_if_env: @@ -1503,13 +1503,13 @@ variations: # - tags: get,generic-python-lib,_package.evaluate # - tags: get,generic-python-lib,_package.absl-py # - tags: get,generic-python-lib,_package.rouge-score - - tags: get,ml-model,whisper,_rclone,_mlc + - tags: get,ml-model,whisper,_r2_downloader,_mlc skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_rclone + - tags: get,dataset,whisper,_preprocessed,_mlc,_r2_downloader names: - whisper-dataset skip_if_env: @@ -1525,7 +1525,7 @@ variations: env: MLC_MODEL: deepseek-r1 deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_rclone + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2_downloader names: - deepseek-r1-preprocessed-dataset skip_if_env: From 326f4090d08efd1ca98d9e360b855215a4abea21 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Jul 2025 22:16:43 +0100 Subject: [PATCH 026/124] Fix docker root run, numactl detect (#534) * Fix docker root run, numactl detect --- .github/workflows/build_wheel_off.yml | 1 - .github/workflows/build_wheels.yml | 1 - script/build-dockerfile/customize.py | 2 +- script/detect-sudo/customize.py | 5 ++++- script/get-generic-sys-util/meta.yaml | 7 +++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_wheel_off.yml b/.github/workflows/build_wheel_off.yml index 199c11d54..4fbc24dba 100644 --- a/.github/workflows/build_wheel_off.yml +++ b/.github/workflows/build_wheel_off.yml @@ -7,7 +7,6 @@ on: paths: - VERSION - jobs: build_wheels: if: github.repository_owner == 'mlcommons' diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 55e79c0ae..86b93ac06 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -5,7 +5,6 @@ on: types: [published] workflow_dispatch: {} - jobs: build_wheels: if: github.repository_owner == 'mlcommons' diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 16685c7a4..93726ffd9 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -264,7 +264,7 @@ def preprocess(i): env['MLC_DOCKER_USE_DEFAULT_USER'] = 'yes' if docker_user and not is_true( - env.get('MLC_DOCKER_USE_DEFAULT_USER', '') and docker_user != 'root'): + env.get('MLC_DOCKER_USE_DEFAULT_USER', '')) and docker_user != 'root': f.write('RUN groupadd -g $GID -o ' + docker_group + EOL) diff --git a/script/detect-sudo/customize.py b/script/detect-sudo/customize.py index 104d8b781..56e6ae627 100644 --- a/script/detect-sudo/customize.py +++ b/script/detect-sudo/customize.py @@ -4,7 +4,6 @@ import subprocess import select import sys -import grp import threading import getpass @@ -13,6 +12,9 @@ def preprocess(i): os_info = i['os_info'] + if os_info['platform'] == 'windows': + return {'return': 0} + env = i['env'] meta = i['meta'] @@ -98,6 +100,7 @@ def prompt_retry(logger, timeout=10, default_retry=False): def is_user_in_sudo_group(logger): + import grp # noqa """Check if the current user is in the 'sudo' group.""" try: sudo_group = grp.getgrnam('sudo').gr_mem diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index d14be0b41..afc39aaec 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -8,6 +8,7 @@ default_env: MLC_SUDO: sudo deps: - tags: detect,os +- tags: detect,sudo env: MLC_GENERIC_SYS_UTIL_INSTALL_NEEDED: 'no' MLC_SYS_UTIL_VERSION_CMD: '' @@ -573,8 +574,6 @@ variations: dnf: libudev-devl yum: libudev-devel linux-tools: - deps: - - tags: detect,os env: MLC_SYS_UTIL_NAME: linux-tools new_env_keys: @@ -641,13 +640,13 @@ variations: tags: install,numactl,from.src env: MLC_SYS_UTIL_NAME: numactl - MLC_TMP_VERSION_DETECT_GROUP_NUMBER: 0 + MLC_SYS_UTIL_CHECK_CMD: 'numactl -s' new_env_keys: - MLC_NUMACTL_VERSION state: numactl: apt: numactl - dnf: numactl-devel + dnf: numactl yum: numactl-devel nvidia-cuda-toolkit: env: From 30ab166fa33f6f97f61e26bb3c075ed694714b0b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 23 Jul 2025 21:17:08 +0000 Subject: [PATCH 027/124] [Automated Commit] Document script/get-generic-sys-util/meta.yaml [skip ci] --- script/get-generic-sys-util/README.md | 136 ++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 script/get-generic-sys-util/README.md diff --git a/script/get-generic-sys-util/README.md b/script/get-generic-sys-util/README.md new file mode 100644 index 000000000..ec30d7828 --- /dev/null +++ b/script/get-generic-sys-util/README.md @@ -0,0 +1,136 @@ +# README for get-generic-sys-util +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +``` +mkdir /mnt/user/MLC +ln -s /mnt/user/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,sys-util,generic,generic-sys-util +``` + +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--fail_safe` | | | `` | +| `--ignore_missing` | | | `` | +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Mode + +- `detect` (default) +- `install` + +### Ungrouped + +- `autoconf` +- `bzip2` +- `cmake` +- `dmidecode` +- `ffmpeg` +- `flex` +- `g++-11` +- `g++-12` +- `g++-9` +- `gcc-11` +- `gcc-9` +- `gflags-dev` +- `git-lfs` +- `glog-dev` +- `ipmitool` +- `libboost-all-dev` +- `libbz2-dev` +- `libev-dev` +- `libffi` +- `libffi-dev` +- `libffi7` +- `libffi8` +- `libgdbm-dev` +- `libgl` +- `libgl1-mesa-glx` +- `libgmock-dev` +- `liblzma-dev` +- `libmkl-dev` +- `libmpfr-dev` +- `libncurses-dev` +- `libnuma-dev` +- `libpci-dev` +- `libpng-dev` +- `libre2-dev` +- `libreadline-dev` +- `libsm6` +- `libsqlite3-dev` +- `libssl-dev` +- `libudev-dev` +- `libxext6` +- `linux-tools` +- `md5sha1sum` +- `ninja-build` +- `nlohmann-json3-dev` +- `ntpdate` +- `numactl` +- `nvidia-cuda-toolkit` +- `pkg-config` +- `postfix` +- `psmisc` +- `rapidjson-dev` +- `rsync` +- `screen` +- `sox` +- `systemd` +- `tk-dev` +- `transmission` +- `unzip` +- `vim-common` +- `wget` +- `wkhtmltopdf` +- `xfonts-base` +- `xz` +- `zlib` +- `zlib1g-dev` From ed8a52b63339ed696f3f6dbcd0df6bd1c49be03e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 23 Jul 2025 23:03:46 +0100 Subject: [PATCH 028/124] Cleanup docs (#535) * Cleanup docs --- script/add-custom-nvidia-system/meta.yaml | 8 ++++---- script/app-image-classification-onnx-py/customize.py | 4 ++-- script/app-image-classification-onnx-py/run.sh | 4 ++-- .../app-image-classification-tf-onnx-cpp/meta.yaml | 2 +- script/app-image-classification-torch-py/run.sh | 2 +- script/app-loadgen-generic-python/meta.yaml | 12 ++++++------ .../app-mlperf-automotive-mlcommons-python/meta.yaml | 12 ++++++------ script/app-mlperf-automotive/customize.py | 5 +---- script/app-mlperf-automotive/meta.yaml | 6 +++--- script/app-mlperf-inference-amd/meta.yaml | 8 ++++---- .../meta.yaml | 2 +- script/app-mlperf-inference-dummy/meta.yaml | 8 ++++---- script/app-mlperf-inference-intel/meta.yaml | 8 ++++---- script/app-mlperf-inference-mlcommons-cpp/meta.yaml | 10 +++++----- .../app-mlperf-inference-mlcommons-python/meta.yaml | 12 ++++++------ script/app-mlperf-inference-nvidia/meta.yaml | 8 ++++---- script/app-mlperf-inference-qualcomm/meta.yaml | 10 +++++----- script/app-mlperf-inference-redhat/meta.yaml | 8 ++++---- script/app-mlperf-inference/customize.py | 6 +++--- script/app-mlperf-inference/meta.yaml | 10 +++++----- script/app-mlperf-training-nvidia/meta.yaml | 8 ++++---- script/app-mlperf-training-reference/meta.yaml | 8 ++++---- script/app-stable-diffusion-onnx-py/meta.yaml | 2 +- script/authenticate-github-cli/run.sh | 2 +- .../run-template.sh | 2 +- script/benchmark-program/customize.py | 2 +- script/benchmark-program/run.sh | 4 ++-- script/build-dockerfile/customize.py | 2 +- .../build-mlperf-inference-server-nvidia/meta.yaml | 10 +++++----- script/calibrate-model-for.qaic/run.sh | 2 +- .../run.sh | 2 +- script/compile-model-for.qaic/run.sh | 2 +- script/convert-csv-to-md/run.sh | 2 +- script/create-custom-cache-entry/customize.py | 2 +- script/create-fpgaconvnet-app-tinyml/run.sh | 2 +- script/create-fpgaconvnet-config-tinyml/run.sh | 2 +- script/detect-sudo/run.sh | 2 +- script/draw-graph-from-json-data/run.sh | 2 +- script/dump-pip-freeze/run.sh | 2 +- script/extract-file/customize.py | 2 +- script/fail/customize.py | 2 +- .../customize.py | 2 +- script/generate-mlperf-inference-user-conf/meta.yaml | 6 +++--- script/generate-mlperf-tiny-report/meta.yaml | 8 ++++---- script/generate-nvidia-engine/meta.yaml | 8 ++++---- script/get-android-sdk/customize.py | 2 +- script/get-aria2/customize.py | 2 +- script/get-cache-dir/meta.yaml | 2 +- script/get-cmsis_5/meta.yaml | 2 +- script/get-cmsis_5/run.sh | 4 ++-- script/get-cuda-devices/run.sh | 2 +- script/get-dataset-cnndm/meta.yaml | 2 +- script/get-dataset-coco/customize.py | 2 +- script/get-dataset-cognata-mlcommons/meta.yaml | 2 +- script/get-dataset-cognata-mlcommons/run.sh | 2 +- script/get-dataset-igbh/run.sh | 2 +- script/get-dataset-imagenet-train/customize.py | 2 +- script/get-dataset-imagenet-val/customize.py | 2 +- script/get-dataset-librispeech/meta.yaml | 2 +- script/get-dataset-nuscenes/run.sh | 2 +- script/get-dataset-squad/meta.yaml | 2 +- script/get-dataset-waymo/run.sh | 2 +- script/get-dlrm-data-mlperf-inference/run.sh | 2 +- script/get-ml-model-pointpainting/run.sh | 2 +- script/get-ml-model-retinanet/run-no-nms.sh | 2 +- script/get-mlperf-automotive-scratch-space/run.sh | 2 +- .../get-mlperf-inference-intel-scratch-space/run.sh | 2 +- .../get-mlperf-inference-nvidia-scratch-space/run.sh | 2 +- script/get-preprocessed-dataset-openorca/meta.yaml | 2 +- script/get-preprocessed-dataset-squad/run-packed.sh | 2 +- script/get-preprocessed-dataset-squad/run.sh | 2 +- script/get-rclone-config/run.sh | 2 +- script/get-rclone/customize.py | 2 +- script/get-rclone/meta.yaml | 2 +- script/get-sys-utils-cm/do_pip_installs.sh.old | 6 ------ .../COPYRIGHT.md | 0 .../customize.py | 2 +- .../do_pip_installs.sh | 0 .../meta.yaml | 2 +- .../requirements.txt | 0 .../run-arch.sh | 0 .../run-debian.sh | 0 .../run-macos.sh | 0 .../run-rhel.sh | 3 ++- .../run-sles.sh | 0 .../run-ubuntu.sh | 0 script/get-xilinx-sdk/run.sh | 2 +- script/install-mlperf-logging-from-src/run.sh | 2 +- script/install-nccl-libs/run.sh | 2 +- script/install-numactl-from-src/meta.yaml | 2 +- script/prepare-training-data-bert/run-nvidia.sh | 2 +- script/prepare-training-data-bert/run-reference.sh | 2 +- script/prepare-training-data-bert/run.sh | 2 +- script/prepare-training-data-resnet/run-nvidia.sh | 2 +- script/prepare-training-data-resnet/run-reference.sh | 2 +- script/prune-bert-models/customize.py | 2 +- .../reproduce-mlperf-octoml-tinyml-results/meta.yaml | 2 +- script/reproduce-mlperf-training-nvidia/meta.yaml | 6 +++--- script/run-all-mlperf-models/run-bert-macos.sh | 2 +- script/run-all-mlperf-models/run-bert.sh | 2 +- .../run-all-mlperf-models/run-cpp-implementation.sh | 2 +- script/run-all-mlperf-models/run-mobilenet-models.sh | 2 +- script/run-all-mlperf-models/run-nvidia-4090.sh | 2 +- script/run-all-mlperf-models/run-nvidia-a100.sh | 2 +- script/run-all-mlperf-models/run-nvidia-t4.sh | 2 +- script/run-all-mlperf-models/run-reference-models.sh | 2 +- script/run-all-mlperf-models/run-resnet50-macos.sh | 2 +- script/run-all-mlperf-models/run-resnet50.sh | 2 +- script/run-all-mlperf-models/run-retinanet-sh | 2 +- script/run-all-mlperf-models/template.sh | 2 +- script/run-docker-container/customize.py | 2 +- script/run-mlperf-inference-app/customize.py | 4 ++-- script/runtime-system-infos/meta.yaml | 8 ++++---- script/set-device-settings-qaic/run.sh | 2 +- script/set-performance-mode/run-ubuntu.sh | 2 +- script/set-performance-mode/run.sh | 2 +- script/set-user-limits/run.sh | 2 +- script/set-venv/customize.py | 4 ++-- 118 files changed, 188 insertions(+), 196 deletions(-) delete mode 100644 script/get-sys-utils-cm/do_pip_installs.sh.old rename script/{get-sys-utils-cm => get-sys-utils-mlc}/COPYRIGHT.md (100%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/customize.py (99%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/do_pip_installs.sh (100%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/meta.yaml (95%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/requirements.txt (100%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/run-arch.sh (100%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/run-debian.sh (100%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/run-macos.sh (100%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/run-rhel.sh (96%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/run-sles.sh (100%) rename script/{get-sys-utils-cm => get-sys-utils-mlc}/run-ubuntu.sh (100%) diff --git a/script/add-custom-nvidia-system/meta.yaml b/script/add-custom-nvidia-system/meta.yaml index 94285d986..01d6b8674 100644 --- a/script/add-custom-nvidia-system/meta.yaml +++ b/script/add-custom-nvidia-system/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: add-custom-nvidia-system uid: b2e6c46c6e8745a3 cache: true @@ -9,7 +9,7 @@ category: "MLPerf benchmark support" docker: real_run: False -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - add - custom @@ -17,7 +17,7 @@ tags: - nvidia -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -28,7 +28,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect python3 - tags: get,python3 diff --git a/script/app-image-classification-onnx-py/customize.py b/script/app-image-classification-onnx-py/customize.py index 39ae86bba..c27248f7d 100644 --- a/script/app-image-classification-onnx-py/customize.py +++ b/script/app-image-classification-onnx-py/customize.py @@ -46,8 +46,8 @@ def postprocess(i): with open(fjson, 'w', encoding='utf-8') as f: json.dump(data, f, ensure_ascii=False, indent=4) except Exception as e: - logger.warning('CM warning: {}'.format(e)) - logger.warning('CM warning: {}'.format(e)) + logger.warning('MLC warning: {}'.format(e)) + logger.warning('MLC warning: {}'.format(e)) try: import yaml diff --git a/script/app-image-classification-onnx-py/run.sh b/script/app-image-classification-onnx-py/run.sh index 4325faf5a..f88018739 100644 --- a/script/app-image-classification-onnx-py/run.sh +++ b/script/app-image-classification-onnx-py/run.sh @@ -11,7 +11,7 @@ fi MLC_PYTHON_BIN=${MLC_PYTHON_BIN_WITH_PATH:-python3} MLC_TMP_CURRENT_SCRIPT_PATH=${MLC_TMP_CURRENT_SCRIPT_PATH:-$PWD} -# connect CM intelligent components with CK env +# connect MLC intelligent components with CK env export CK_ENV_ONNX_MODEL_ONNX_FILEPATH=${MLC_ML_MODEL_FILE_WITH_PATH} export CK_ENV_ONNX_MODEL_INPUT_LAYER_NAME="input_tensor:0" export CK_ENV_ONNX_MODEL_OUTPUT_LAYER_NAME="softmax_tensor:0" @@ -33,5 +33,5 @@ echo "" ${MLC_PYTHON_BIN} ${MLC_TMP_CURRENT_SCRIPT_PATH}/src/onnx_classify.py test $? -eq 0 || exit 1 -# Just a demo to pass environment variables from native scripts back to CM workflows +# Just a demo to pass environment variables from native scripts back to MLC workflows echo "MLC_APP_IMAGE_CLASSIFICATION_ONNX_PY=sucess" > tmp-run-env.out diff --git a/script/app-image-classification-tf-onnx-cpp/meta.yaml b/script/app-image-classification-tf-onnx-cpp/meta.yaml index 957a0d28f..b6afbd6cc 100644 --- a/script/app-image-classification-tf-onnx-cpp/meta.yaml +++ b/script/app-image-classification-tf-onnx-cpp/meta.yaml @@ -7,7 +7,7 @@ default_env: MLC_BATCH_SIZE: '1' deps: - tags: detect,os -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - tags: get,gcc - tags: get,dataset,image-classification,original - tags: get,dataset-aux,image-classification diff --git a/script/app-image-classification-torch-py/run.sh b/script/app-image-classification-torch-py/run.sh index 478332299..f5ce00034 100644 --- a/script/app-image-classification-torch-py/run.sh +++ b/script/app-image-classification-torch-py/run.sh @@ -2,7 +2,7 @@ MLC_TMP_CURRENT_SCRIPT_PATH=${MLC_TMP_CURRENT_SCRIPT_PATH:-$PWD} -# connect CM intelligent components with CK env +# connect MLC intelligent components with CK env export MLC_ML_TORCH_MODEL_NAME=resnet50 export MLC_ML_MODEL_INPUT_DATA_TYPE=float32 export MLC_ML_MODEL_IMAGE_HEIGHT=224 diff --git a/script/app-loadgen-generic-python/meta.yaml b/script/app-loadgen-generic-python/meta.yaml index ba5a3c616..16f564a59 100644 --- a/script/app-loadgen-generic-python/meta.yaml +++ b/script/app-loadgen-generic-python/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-loadgen-generic-python uid: d3d949cc361747a6 @@ -10,7 +10,7 @@ category: "Modular MLPerf inference benchmark pipeline" developers: "[Gaz Iqbal](https://www.linkedin.com/in/gaziqbal), [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - loadgen @@ -49,7 +49,7 @@ input_mapping: new_env_keys: - MLC_MLPERF_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -82,7 +82,7 @@ deps: - loadgen ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC # ONNX - enable_if_env: MLC_MLPERF_BACKEND: @@ -110,7 +110,7 @@ deps: - onnx ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC # PyTorch # CPU @@ -175,7 +175,7 @@ deps: -# Customize this CM script +# Customize this MLC script variations: pytorch: diff --git a/script/app-mlperf-automotive-mlcommons-python/meta.yaml b/script/app-mlperf-automotive-mlcommons-python/meta.yaml index c574cc7c0..1a3d8d6d2 100644 --- a/script/app-mlperf-automotive-mlcommons-python/meta.yaml +++ b/script/app-mlperf-automotive-mlcommons-python/meta.yaml @@ -7,7 +7,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf inference benchmark pipeline for ABTF model" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - automotive - mlcommons @@ -65,7 +65,7 @@ input_mapping: multistream_target_latency: MLC_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY output: MLC_MLPERF_OUTPUT_DIR -# Duplicate CM environment variables to the ones used in native apps +# Duplicate MLC environment variables to the ones used in native apps env_key_mappings: MLC_HOST_: HOST_ MLC_ML_: ML_ @@ -87,7 +87,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features @@ -97,7 +97,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python @@ -105,7 +105,7 @@ deps: - python - python3 - # Use cmind inside CM scripts + # Use cmind inside MLC scripts - tags: get,generic-python-lib,_package.cmind @@ -120,7 +120,7 @@ deps: ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC ## Onnx CPU Runtime - tags: get,generic-python-lib,_onnxruntime diff --git a/script/app-mlperf-automotive/customize.py b/script/app-mlperf-automotive/customize.py index 32f5b3416..45cf6da18 100644 --- a/script/app-mlperf-automotive/customize.py +++ b/script/app-mlperf-automotive/customize.py @@ -283,7 +283,7 @@ def postprocess(i): if env.get('MLC_HOST_SYSTEM_NAME', '') != '': host_info['system_name'] = env['MLC_HOST_SYSTEM_NAME'] - # Check CM automation repository + # Check MLC automation repository repo_name = 'mlcommons@mlperf-automations' repo_hash = '' r = mlc.access({'action': 'find', 'automation': 'repo', @@ -293,9 +293,6 @@ def postprocess(i): if os.path.isdir(repo_path): repo_name = os.path.basename(repo_path) - # Check dev - # if repo_name == 'cm4mlops': repo_name = 'mlcommons@cm4mlops' - r = utils.run_system_cmd({ 'path': repo_path, 'cmd': 'git rev-parse HEAD'}) diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index ee4af5c5b..3afa3a69e 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -7,7 +7,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf automotive benchmark pipeline for ABTF models" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - app-mlperf-inference @@ -72,7 +72,7 @@ new_env_keys: new_state_keys: - mlc-mlperf-inference-results* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features @@ -82,7 +82,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python diff --git a/script/app-mlperf-inference-amd/meta.yaml b/script/app-mlperf-inference-amd/meta.yaml index 082408f87..e3f9c8789 100644 --- a/script/app-mlperf-inference-amd/meta.yaml +++ b/script/app-mlperf-inference-amd/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-amd uid: 467cdb20aabc4394 cache: false @@ -9,7 +9,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -68,7 +68,7 @@ new_env_keys: - MLC_SQUAD_ACCURACY_DTYPE -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -79,7 +79,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc ######################################################################## diff --git a/script/app-mlperf-inference-ctuning-cpp-tflite/meta.yaml b/script/app-mlperf-inference-ctuning-cpp-tflite/meta.yaml index 68e482375..8a7446499 100644 --- a/script/app-mlperf-inference-ctuning-cpp-tflite/meta.yaml +++ b/script/app-mlperf-inference-ctuning-cpp-tflite/meta.yaml @@ -21,7 +21,7 @@ default_env: deps: - tags: detect,os - tags: detect,cpu -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - enable_if_env: MLC_MLPERF_DEVICE: - gpu diff --git a/script/app-mlperf-inference-dummy/meta.yaml b/script/app-mlperf-inference-dummy/meta.yaml index b671994b7..28ce530aa 100644 --- a/script/app-mlperf-inference-dummy/meta.yaml +++ b/script/app-mlperf-inference-dummy/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-dummy uid: 5b71627383a94576 cache: false @@ -9,7 +9,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -68,7 +68,7 @@ new_env_keys: - MLC_SQUAD_ACCURACY_DTYPE -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -79,7 +79,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc ######################################################################## diff --git a/script/app-mlperf-inference-intel/meta.yaml b/script/app-mlperf-inference-intel/meta.yaml index 823ae9343..aba29ceef 100644 --- a/script/app-mlperf-inference-intel/meta.yaml +++ b/script/app-mlperf-inference-intel/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-intel uid: c05a90433bb04cc1 cache: false @@ -10,7 +10,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -68,7 +68,7 @@ new_state_keys: -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -79,7 +79,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Get MLPerf logging library diff --git a/script/app-mlperf-inference-mlcommons-cpp/meta.yaml b/script/app-mlperf-inference-mlcommons-cpp/meta.yaml index 0763a13e8..fa1a5a692 100644 --- a/script/app-mlperf-inference-mlcommons-cpp/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-cpp/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-mlcommons-cpp uid: bf62405e6c7a44bf @@ -9,7 +9,7 @@ category: "Modular MLPerf inference benchmark pipeline" developers: "[Thomas Zhu](https://www.linkedin.com/in/hanwen-zhu-483614189), [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - mlcommons @@ -48,7 +48,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -59,7 +59,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect CUDA if required - tags: get,cuda,_cudnn @@ -81,7 +81,7 @@ deps: - inference-src ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC - enable_if_env: MLC_MLPERF_BACKEND: - onnxruntime diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 2ea27873b..c5ebee492 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-mlcommons-python uid: ff149e9781fc4b65 @@ -9,7 +9,7 @@ category: "Modular MLPerf inference benchmark pipeline" developers: "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Thomas Zhu](https://www.linkedin.com/in/hanwen-zhu-483614189), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - vision @@ -70,7 +70,7 @@ input_mapping: deeplab_resnet50_path: MLC_ML_MODEL_DPLAB_RESNET50_PATH waymo_path: MLC_DATASET_WAYMO_PATH -# Duplicate CM environment variables to the ones used in native apps +# Duplicate MLC environment variables to the ones used in native apps env_key_mappings: MLC_HOST_: HOST_ MLC_ML_: ML_ @@ -89,7 +89,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features - tags: detect,os @@ -98,7 +98,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python @@ -126,7 +126,7 @@ deps: - tensorrt ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC ## Onnx CPU Runtime - tags: get,generic-python-lib,_onnxruntime diff --git a/script/app-mlperf-inference-nvidia/meta.yaml b/script/app-mlperf-inference-nvidia/meta.yaml index b152945ed..6dda8d4a5 100644 --- a/script/app-mlperf-inference-nvidia/meta.yaml +++ b/script/app-mlperf-inference-nvidia/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-nvidia uid: bc3b17fb430f4732 cache: false @@ -10,7 +10,7 @@ automation_uid: 5b4e0237da074764 category: "Reproduce MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -92,7 +92,7 @@ input_mapping: embedding_weights_on_gpu_part: MLC_MLPERF_NVIDIA_HARNESS_EMBEDDING_WEIGHTS_ON_GPU_PART sdxl_batcher_time_limit: MLC_MLPERF_NVIDIA_HARNESS_SDXL_SERVER_BATCHER_TIME_LIMIT -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -103,7 +103,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Get Nvidia scratch space where data and models get downloaded - tags: get,mlperf,inference,nvidia,scratch,space diff --git a/script/app-mlperf-inference-qualcomm/meta.yaml b/script/app-mlperf-inference-qualcomm/meta.yaml index 246d89098..b4309881c 100644 --- a/script/app-mlperf-inference-qualcomm/meta.yaml +++ b/script/app-mlperf-inference-qualcomm/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-qualcomm uid: eef1aca5d7c0470e cache: false @@ -10,7 +10,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -79,7 +79,7 @@ new_env_keys: - MLC_SQUAD_ACCURACY_DTYPE -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -90,7 +90,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc - tags: get,git,repo names: @@ -226,7 +226,7 @@ deps: ######################################################################## - # Install ML engines via CM + # Install ML engines via MLC - enable_if_env: MLC_MLPERF_BACKEND: - onnxruntime diff --git a/script/app-mlperf-inference-redhat/meta.yaml b/script/app-mlperf-inference-redhat/meta.yaml index cec71335e..452e8094e 100644 --- a/script/app-mlperf-inference-redhat/meta.yaml +++ b/script/app-mlperf-inference-redhat/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference-redhat uid: 82c9bb3c222447ca cache: false @@ -9,7 +9,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -68,7 +68,7 @@ new_env_keys: - MLC_SQUAD_ACCURACY_DTYPE -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -79,7 +79,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc ######################################################################## diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index 37f5ca9b2..62c05a287 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -395,7 +395,7 @@ def postprocess(i): if env.get('MLC_HOST_SYSTEM_NAME', '') != '': host_info['system_name'] = env['MLC_HOST_SYSTEM_NAME'] - # Check CM automation repository + # Check MLC automation repository repo_name = 'mlcommons@mlperf-automations' repo_hash = '' r = mlc.access({'action': 'find', 'automation': 'repo', @@ -429,7 +429,7 @@ def postprocess(i): cmd = "" xcmd = "" - readme_init = "*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.*\n\n" + readme_init = "*Check [MLC MLPerf docs](https://docs.mlcommons.org/inference) for more details.*\n\n" readme_body = "## Host platform\n\n* OS version: {}\n* CPU version: {}\n* Python version: {}\n* MLC version: {}\n\n".format(platform.platform(), platform.processor(), sys.version, mlc_version) @@ -438,7 +438,7 @@ def postprocess(i): if repo_hash != '': x += ' --checkout=' + str(repo_hash) - readme_body += "## CM Run Command\n\nSee [CM installation guide](https://docs.mlcommons.org/inference/install/).\n\n" + \ + readme_body += "## MLC Run Command\n\nSee [MLC installation guide](https://docs.mlcommons.org/inference/install/).\n\n" + \ "```bash\npip install -U mlcflow\n\nmlc rm cache -f\n\nmlc pull repo {}\n\n{}\n```".format( x, xcmd) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index bbcc99fa8..b1d7ef38e 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-inference uid: d775cac873ee4231 @@ -9,7 +9,7 @@ category: "Modular MLPerf inference benchmark pipeline" developers: "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Thomas Zhu](https://www.linkedin.com/in/hanwen-zhu-483614189), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - vision @@ -68,7 +68,7 @@ input_mapping: predeps: False -# Duplicate CM environment variables to the ones used in native apps +# Duplicate MLC environment variables to the ones used in native apps env_key_mappings: MLC_HOST_: HOST_ MLC_ML_: ML_ @@ -82,14 +82,14 @@ new_state_keys: - app_mlperf_inference_* - mlc-mlperf-inference-results* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features - tags: detect,os # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python diff --git a/script/app-mlperf-training-nvidia/meta.yaml b/script/app-mlperf-training-nvidia/meta.yaml index abf4e7dd9..f4aa6eea2 100644 --- a/script/app-mlperf-training-nvidia/meta.yaml +++ b/script/app-mlperf-training-nvidia/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-training-nvidia uid: 1e2e357618cc4674 @@ -7,7 +7,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf training benchmark pipeline" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - vision @@ -43,7 +43,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features @@ -53,7 +53,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python diff --git a/script/app-mlperf-training-reference/meta.yaml b/script/app-mlperf-training-reference/meta.yaml index 45b0633be..ec2666af2 100644 --- a/script/app-mlperf-training-reference/meta.yaml +++ b/script/app-mlperf-training-reference/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: app-mlperf-training-reference uid: 0c4b11bdcf494b4f @@ -7,7 +7,7 @@ automation_uid: 5b4e0237da074764 category: "Modular MLPerf training benchmark pipeline" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - app - vision @@ -45,7 +45,7 @@ new_state_keys: - mlperf-inference-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features @@ -55,7 +55,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect/install python - tags: get,python diff --git a/script/app-stable-diffusion-onnx-py/meta.yaml b/script/app-stable-diffusion-onnx-py/meta.yaml index 4aacbe801..978d245bb 100644 --- a/script/app-stable-diffusion-onnx-py/meta.yaml +++ b/script/app-stable-diffusion-onnx-py/meta.yaml @@ -20,7 +20,7 @@ tags_help: "modular python app stable-diffusion onnx" deps: - tags: detect,os -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - names: - python - python3 diff --git a/script/authenticate-github-cli/run.sh b/script/authenticate-github-cli/run.sh index ad1472f09..62f121083 100644 --- a/script/authenticate-github-cli/run.sh +++ b/script/authenticate-github-cli/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/benchmark-any-mlperf-inference-implementation/run-template.sh b/script/benchmark-any-mlperf-inference-implementation/run-template.sh index 8e0cb42c0..b77849229 100644 --- a/script/benchmark-any-mlperf-inference-implementation/run-template.sh +++ b/script/benchmark-any-mlperf-inference-implementation/run-template.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/benchmark-program/customize.py b/script/benchmark-program/customize.py index be97afe39..66886971a 100644 --- a/script/benchmark-program/customize.py +++ b/script/benchmark-program/customize.py @@ -106,7 +106,7 @@ def preprocess(i): # Print info logger.info( '***************************************************************************') - logger.info('CM script::benchmark-program/run.sh') + logger.info('MLC script::benchmark-program/run.sh') logger.info('') logger.info('Run Directory: {}'.format(env.get('MLC_RUN_DIR', ''))) diff --git a/script/benchmark-program/run.sh b/script/benchmark-program/run.sh index 6caace406..adad9f555 100755 --- a/script/benchmark-program/run.sh +++ b/script/benchmark-program/run.sh @@ -29,7 +29,7 @@ cd "${MLC_RUN_DIR}" if [[ "${MLC_DEBUG_SCRIPT_BENCHMARK_PROGRAM}" == "True" ]]; then echo "*****************************************************" - echo "You are now in Debug shell with pre-set CM env and can run the following command line manually:" + echo "You are now in Debug shell with pre-set MLC env and can run the following command line manually:" echo "" if [[ "${MLC_RUN_CMD0}" != "" ]]; then @@ -39,7 +39,7 @@ if [[ "${MLC_DEBUG_SCRIPT_BENCHMARK_PROGRAM}" == "True" ]]; then fi echo "" - echo "Type exit to return to CM script." + echo "Type exit to return to MLC script." echo "" # echo "You can also run . ./debug-script-benchmark-program.sh to reproduce and customize run." # echo "" diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 93726ffd9..5b34a3f6c 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -15,7 +15,7 @@ def preprocess(i): if env["MLC_DOCKER_OS"] not in ["ubuntu", "rhel", "arch"]: return { - 'return': 1, 'error': f"Specified docker OS: {env['MLC_DOCKER_OS']}. Currently only ubuntu, rhel and arch are supported in CM docker"} + 'return': 1, 'error': f"Specified docker OS: {env['MLC_DOCKER_OS']}. Currently only ubuntu, rhel and arch are supported in MLC docker"} path = i['run_script_input']['path'] diff --git a/script/build-mlperf-inference-server-nvidia/meta.yaml b/script/build-mlperf-inference-server-nvidia/meta.yaml index 7fc8f9b71..8c57fd809 100644 --- a/script/build-mlperf-inference-server-nvidia/meta.yaml +++ b/script/build-mlperf-inference-server-nvidia/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: build-mlperf-inference-server-nvidia uid: f37403af5e9f4541 cache: true @@ -9,7 +9,7 @@ default_version: r3.1 category: "MLPerf benchmark support" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - build - mlcommons @@ -33,7 +33,7 @@ input_mapping: custom_system: MLC_CUSTOM_SYSTEM_NVIDIA clean: MLC_MAKE_CLEAN -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -44,7 +44,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect python3 - tags: get,python3 @@ -76,7 +76,7 @@ deps: # Detect gcc - tags: get,gcc - # Detect CMake + # Detect MLCake - tags: get,cmake version: "3.25.1" diff --git a/script/calibrate-model-for.qaic/run.sh b/script/calibrate-model-for.qaic/run.sh index 7da7962b9..d643e811e 100644 --- a/script/calibrate-model-for.qaic/run.sh +++ b/script/calibrate-model-for.qaic/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/clean-nvidia-mlperf-inference-scratch-space/run.sh b/script/clean-nvidia-mlperf-inference-scratch-space/run.sh index 32cf4d51e..4a22ce805 100644 --- a/script/clean-nvidia-mlperf-inference-scratch-space/run.sh +++ b/script/clean-nvidia-mlperf-inference-scratch-space/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/compile-model-for.qaic/run.sh b/script/compile-model-for.qaic/run.sh index d20c3a705..6ce29fde0 100644 --- a/script/compile-model-for.qaic/run.sh +++ b/script/compile-model-for.qaic/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/convert-csv-to-md/run.sh b/script/convert-csv-to-md/run.sh index 7da7962b9..d643e811e 100644 --- a/script/convert-csv-to-md/run.sh +++ b/script/convert-csv-to-md/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/create-custom-cache-entry/customize.py b/script/create-custom-cache-entry/customize.py index 795235bfd..0f275b0a5 100644 --- a/script/create-custom-cache-entry/customize.py +++ b/script/create-custom-cache-entry/customize.py @@ -5,7 +5,7 @@ def preprocess(i): - # CM script internal variables + # MLC script internal variables env = i['env'] logger = i['automation'].logger extra_cache_tags = [] diff --git a/script/create-fpgaconvnet-app-tinyml/run.sh b/script/create-fpgaconvnet-app-tinyml/run.sh index 35de74bab..65f2a08f9 100644 --- a/script/create-fpgaconvnet-app-tinyml/run.sh +++ b/script/create-fpgaconvnet-app-tinyml/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/create-fpgaconvnet-config-tinyml/run.sh b/script/create-fpgaconvnet-config-tinyml/run.sh index 35de74bab..65f2a08f9 100644 --- a/script/create-fpgaconvnet-config-tinyml/run.sh +++ b/script/create-fpgaconvnet-config-tinyml/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/detect-sudo/run.sh b/script/detect-sudo/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/detect-sudo/run.sh +++ b/script/detect-sudo/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/draw-graph-from-json-data/run.sh b/script/draw-graph-from-json-data/run.sh index 32cf4d51e..4a22ce805 100644 --- a/script/draw-graph-from-json-data/run.sh +++ b/script/draw-graph-from-json-data/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/dump-pip-freeze/run.sh b/script/dump-pip-freeze/run.sh index 0e379ea14..a6b4b8dea 100644 --- a/script/dump-pip-freeze/run.sh +++ b/script/dump-pip-freeze/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index ddefc0757..f6b185fe0 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -36,7 +36,7 @@ def preprocess(i): env['MLC_EXTRACT_FILENAME'] = filename - # Check if extract to some path outside CM cache (to reuse large files + # Check if extract to some path outside MLC cache (to reuse large files # later if cache is cleaned) extract_path = env.get('MLC_EXTRACT_PATH', '') if extract_path != '': diff --git a/script/fail/customize.py b/script/fail/customize.py index 6d3eda24d..840aabb82 100644 --- a/script/fail/customize.py +++ b/script/fail/customize.py @@ -19,7 +19,7 @@ def preprocess(i): if is_true(env.get('MLC_FAIL_WINDOWS', '')): if os_info['platform'] == 'windows': return {'return': 1, - 'error': 'CM detected fail condition: running on Windows'} + 'error': 'MLC detected fail condition: running on Windows'} return {'return': 0} diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 875a12906..f6490a31c 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -488,7 +488,7 @@ def generate_submission(env, state, inp, submission_division, logger): result_mode_path, "system_meta.json") else: logger.error( - "WARNING: system_meta.json was not found in the SUT root or mode directory inside the results folder. CM is automatically creating one using the system defaults. Please modify them as required.") + "WARNING: system_meta.json was not found in the SUT root or mode directory inside the results folder. MLC is automatically creating one using the system defaults. Please modify them as required.") if os.path.exists(saved_system_meta_file_path): with open(saved_system_meta_file_path, "r") as f: saved_system_meta = json.load(f) diff --git a/script/generate-mlperf-inference-user-conf/meta.yaml b/script/generate-mlperf-inference-user-conf/meta.yaml index b500b8f94..086de0bb9 100644 --- a/script/generate-mlperf-inference-user-conf/meta.yaml +++ b/script/generate-mlperf-inference-user-conf/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: generate-mlperf-inference-user-conf uid: 3af4475745964b93 @@ -9,7 +9,7 @@ category: "MLPerf benchmark support" developers: "[Arjun Suresh](https://www.linkedin.com/in/arjunsuresh), [Thomas Zhu](https://www.linkedin.com/in/hanwen-zhu-483614189), [Grigori Fursin](https://cKnowledge.org/gfursin)" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - generate - mlperf @@ -63,7 +63,7 @@ new_env_keys: new_state_keys: - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features diff --git a/script/generate-mlperf-tiny-report/meta.yaml b/script/generate-mlperf-tiny-report/meta.yaml index 467226c1b..39de344f6 100644 --- a/script/generate-mlperf-tiny-report/meta.yaml +++ b/script/generate-mlperf-tiny-report/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: generate-mlperf-tiny-report uid: 709c3f3f9b3e4783 @@ -12,7 +12,7 @@ developers: "[Grigori Fursin](https://cKnowledge.org/gfursin)" default_env: MLC_IMPORT_TINYMLPERF_REPO_TAGS: "1.1-private" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - generate - mlperf @@ -23,14 +23,14 @@ tags: input_mapping: repo_tags: MLC_IMPORT_TINYMLPERF_REPO_TAGS -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: # Detect host OS features - tags: detect,os # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect python3 - tags: get,python3 diff --git a/script/generate-nvidia-engine/meta.yaml b/script/generate-nvidia-engine/meta.yaml index b63ba77e1..d6c50605f 100644 --- a/script/generate-nvidia-engine/meta.yaml +++ b/script/generate-nvidia-engine/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: generate-nvidia-engine uid: 0eef9f05b272401f @@ -8,7 +8,7 @@ automation_uid: 5b4e0237da074764 category: "MLPerf benchmark support" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - generate - engine @@ -32,7 +32,7 @@ new_env_keys: - MLC_MLPERF_* - MLC_DATASET_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -43,7 +43,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect CUDA - tags: get,cuda,_cudnn diff --git a/script/get-android-sdk/customize.py b/script/get-android-sdk/customize.py index eedcb31e6..8087f07e3 100644 --- a/script/get-android-sdk/customize.py +++ b/script/get-android-sdk/customize.py @@ -20,7 +20,7 @@ def preprocess(i): # Check if ANDROID_HOME is already set android_home = os.environ.get('ANDROID_HOME', '').strip() - # We are inside CM cache entry + # We are inside MLC cache entry cur_dir = os.getcwd() if android_home == '': diff --git a/script/get-aria2/customize.py b/script/get-aria2/customize.py index 2f7459cde..dfbdafc36 100644 --- a/script/get-aria2/customize.py +++ b/script/get-aria2/customize.py @@ -5,7 +5,7 @@ def preprocess(i): - # Pre-set by CM + # Pre-set by MLC os_info = i['os_info'] env = i['env'] logger = i['automation'].logger diff --git a/script/get-cache-dir/meta.yaml b/script/get-cache-dir/meta.yaml index e02b9a7cb..25aad5eae 100644 --- a/script/get-cache-dir/meta.yaml +++ b/script/get-cache-dir/meta.yaml @@ -2,7 +2,7 @@ alias: get-cache-dir automation_alias: script automation_uid: 5b4e0237da074764 cache: true -category: CM Interface +category: MLC Interface deps: [] docker: run: false diff --git a/script/get-cmsis_5/meta.yaml b/script/get-cmsis_5/meta.yaml index 95ac1ef3a..b35872112 100644 --- a/script/get-cmsis_5/meta.yaml +++ b/script/get-cmsis_5/meta.yaml @@ -11,7 +11,7 @@ default_version: custom deps: - tags: detect,os new_env_keys: -- CMSIS* +- MLCSIS* tags: - get - cmsis diff --git a/script/get-cmsis_5/run.sh b/script/get-cmsis_5/run.sh index 47d1e2554..af857a9a5 100644 --- a/script/get-cmsis_5/run.sh +++ b/script/get-cmsis_5/run.sh @@ -7,11 +7,11 @@ echo "******************************************************" if [ ! -d "cmsis" ]; then if [ -z ${MLC_GIT_SHA} ]; then - echo "Cloning CMSIS_5 from ${MLC_GIT_URL} with branch ${MLC_GIT_CHECKOUT} ${MLC_GIT_DEPTH} ${MLC_GIT_RECURSE_SUBMODULES}..." + echo "Cloning MLCSIS_5 from ${MLC_GIT_URL} with branch ${MLC_GIT_CHECKOUT} ${MLC_GIT_DEPTH} ${MLC_GIT_RECURSE_SUBMODULES}..." git clone ${MLC_GIT_RECURSE_SUBMODULES} -b "${MLC_GIT_CHECKOUT}" ${MLC_GIT_URL} ${MLC_GIT_DEPTH} cmsis if [ "${?}" != "0" ]; then exit 1; fi else - echo "Cloning CMSIS_5 from ${MLC_GIT_URL} with default branch and checkout ${MLC_GIT_CHECKOUT} ${MLC_GIT_DEPTH} ${MLC_GIT_RECURSE_SUBMODULES}..." + echo "Cloning MLCSIS_5 from ${MLC_GIT_URL} with default branch and checkout ${MLC_GIT_CHECKOUT} ${MLC_GIT_DEPTH} ${MLC_GIT_RECURSE_SUBMODULES}..." git clone ${MLC_GIT_RECURSE_SUBMODULES} ${MLC_GIT_URL} ${MLC_GIT_DEPTH} cmsis if [ "${?}" != "0" ]; then exit 1; fi cd cmsis diff --git a/script/get-cuda-devices/run.sh b/script/get-cuda-devices/run.sh index 2ee43d856..c802ec930 100644 --- a/script/get-cuda-devices/run.sh +++ b/script/get-cuda-devices/run.sh @@ -23,7 +23,7 @@ cd ${MLC_TMP_CURRENT_SCRIPT_PATH} ${MLC_NVCC_BIN_WITH_PATH} -allow-unsupported-compiler print_cuda_devices.cu test $? -eq 0 || exit 1 -# Return to the original path obtained in CM +# Return to the original path obtained in MLC echo "" echo "Running program ..." diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index cf70343e3..1e891215c 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -6,7 +6,7 @@ category: AI/ML datasets default_env: MLC_DATASET_CALIBRATION: 'no' deps: -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - names: - python - python3 diff --git a/script/get-dataset-coco/customize.py b/script/get-dataset-coco/customize.py index e8798fbaf..8e904e264 100644 --- a/script/get-dataset-coco/customize.py +++ b/script/get-dataset-coco/customize.py @@ -6,7 +6,7 @@ def preprocess(i): - # CM script internal variables + # MLC script internal variables variation_tags = i.get('variation_tags', []) automation = i['automation'] env = i['env'] diff --git a/script/get-dataset-cognata-mlcommons/meta.yaml b/script/get-dataset-cognata-mlcommons/meta.yaml index ecb10799a..0b24c75e9 100644 --- a/script/get-dataset-cognata-mlcommons/meta.yaml +++ b/script/get-dataset-cognata-mlcommons/meta.yaml @@ -43,7 +43,7 @@ env: deps: -# Prepare dummy CM cache entry to manage dataset +# Prepare dummy MLC cache entry to manage dataset - names: - custom-cache-entry-mlcommons-cognata-dataset tags: create,custom,cache,entry diff --git a/script/get-dataset-cognata-mlcommons/run.sh b/script/get-dataset-cognata-mlcommons/run.sh index 5563a95da..309cd36ba 100644 --- a/script/get-dataset-cognata-mlcommons/run.sh +++ b/script/get-dataset-cognata-mlcommons/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-dataset-igbh/run.sh b/script/get-dataset-igbh/run.sh index edb705045..ba4a04a7c 100644 --- a/script/get-dataset-igbh/run.sh +++ b/script/get-dataset-igbh/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-dataset-imagenet-train/customize.py b/script/get-dataset-imagenet-train/customize.py index 1fe0b8b43..34241f2c9 100644 --- a/script/get-dataset-imagenet-train/customize.py +++ b/script/get-dataset-imagenet-train/customize.py @@ -27,7 +27,7 @@ def preprocess(i): return {'return': 0} else: - return {'return': 1, 'error': 'Please rerun the last CM command with --env.IMAGENET_TRAIN_PATH={path the folder containing full ImageNet training images} or envoke mlcr "get train dataset imagenet" --input={path to the folder containing ImageNet training images}'} + return {'return': 1, 'error': 'Please rerun the last MLC command with --env.IMAGENET_TRAIN_PATH={path the folder containing full ImageNet training images} or envoke mlcr "get train dataset imagenet" --input={path to the folder containing ImageNet training images}'} elif not os.path.isdir(path): if path.endswith(".tar"): diff --git a/script/get-dataset-imagenet-val/customize.py b/script/get-dataset-imagenet-val/customize.py index 9b4569d15..3d4506131 100644 --- a/script/get-dataset-imagenet-val/customize.py +++ b/script/get-dataset-imagenet-val/customize.py @@ -40,7 +40,7 @@ def preprocess(i): env['MLC_DATASET_IMAGENET_VAL_REQUIRE_DAE'] = 'yes' return {'return': 0} - # return {'return':1, 'error':'Please rerun the last CM command + # return {'return':1, 'error':'Please rerun the last MLC command # with --env.IMAGENET_PATH={path the folder containing full # ImageNet images} or envoke mlcr "get val dataset # imagenet" --input={path to the folder containing ImageNet diff --git a/script/get-dataset-librispeech/meta.yaml b/script/get-dataset-librispeech/meta.yaml index c696b3e25..817c0f119 100644 --- a/script/get-dataset-librispeech/meta.yaml +++ b/script/get-dataset-librispeech/meta.yaml @@ -7,7 +7,7 @@ default_version: dev-clean deps: - names: - sys-utils - tags: get,sys-utils-cm + tags: get,sys-utils-mlc env: MLC_DATASET: LIBRISPEECH MLC_WGET_URL: http://www.openslr.org/resources/12/<<>> diff --git a/script/get-dataset-nuscenes/run.sh b/script/get-dataset-nuscenes/run.sh index abe5a17c7..ca6f48e58 100644 --- a/script/get-dataset-nuscenes/run.sh +++ b/script/get-dataset-nuscenes/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-dataset-squad/meta.yaml b/script/get-dataset-squad/meta.yaml index cc55e3b50..10a34b711 100644 --- a/script/get-dataset-squad/meta.yaml +++ b/script/get-dataset-squad/meta.yaml @@ -5,7 +5,7 @@ cache: true category: AI/ML datasets default_version: '1.1' deps: -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc env: MLC_DATASET: SQUAD new_env_keys: diff --git a/script/get-dataset-waymo/run.sh b/script/get-dataset-waymo/run.sh index ba1412ac9..fa7001aad 100644 --- a/script/get-dataset-waymo/run.sh +++ b/script/get-dataset-waymo/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-dlrm-data-mlperf-inference/run.sh b/script/get-dlrm-data-mlperf-inference/run.sh index 180056e2f..58b536f0c 100644 --- a/script/get-dlrm-data-mlperf-inference/run.sh +++ b/script/get-dlrm-data-mlperf-inference/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-ml-model-pointpainting/run.sh b/script/get-ml-model-pointpainting/run.sh index 3197bb8ad..238a89269 100644 --- a/script/get-ml-model-pointpainting/run.sh +++ b/script/get-ml-model-pointpainting/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-ml-model-retinanet/run-no-nms.sh b/script/get-ml-model-retinanet/run-no-nms.sh index 82e546f62..aa0d60151 100644 --- a/script/get-ml-model-retinanet/run-no-nms.sh +++ b/script/get-ml-model-retinanet/run-no-nms.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-mlperf-automotive-scratch-space/run.sh b/script/get-mlperf-automotive-scratch-space/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/get-mlperf-automotive-scratch-space/run.sh +++ b/script/get-mlperf-automotive-scratch-space/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-mlperf-inference-intel-scratch-space/run.sh b/script/get-mlperf-inference-intel-scratch-space/run.sh index 4260c33a9..52d73d4c1 100644 --- a/script/get-mlperf-inference-intel-scratch-space/run.sh +++ b/script/get-mlperf-inference-intel-scratch-space/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-mlperf-inference-nvidia-scratch-space/run.sh b/script/get-mlperf-inference-nvidia-scratch-space/run.sh index 4260c33a9..52d73d4c1 100644 --- a/script/get-mlperf-inference-nvidia-scratch-space/run.sh +++ b/script/get-mlperf-inference-nvidia-scratch-space/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-preprocessed-dataset-openorca/meta.yaml b/script/get-preprocessed-dataset-openorca/meta.yaml index 8ddb5f049..5e9f1a83d 100644 --- a/script/get-preprocessed-dataset-openorca/meta.yaml +++ b/script/get-preprocessed-dataset-openorca/meta.yaml @@ -7,7 +7,7 @@ category_sort: 8500 default_env: MLC_DATASET_CALIBRATION: 'no' deps: -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - names: - python - python3 diff --git a/script/get-preprocessed-dataset-squad/run-packed.sh b/script/get-preprocessed-dataset-squad/run-packed.sh index f42dedcf8..6fd16c7a2 100644 --- a/script/get-preprocessed-dataset-squad/run-packed.sh +++ b/script/get-preprocessed-dataset-squad/run-packed.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-preprocessed-dataset-squad/run.sh b/script/get-preprocessed-dataset-squad/run.sh index 32c02c034..79e8a78d5 100644 --- a/script/get-preprocessed-dataset-squad/run.sh +++ b/script/get-preprocessed-dataset-squad/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-rclone-config/run.sh b/script/get-rclone-config/run.sh index 32cf4d51e..4a22ce805 100644 --- a/script/get-rclone-config/run.sh +++ b/script/get-rclone-config/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/get-rclone/customize.py b/script/get-rclone/customize.py index 91fa9b685..eebd76c71 100644 --- a/script/get-rclone/customize.py +++ b/script/get-rclone/customize.py @@ -146,7 +146,7 @@ def postprocess(i): cur_dir = os.getcwd() path_bin = os.path.join(cur_dir, file_name) if os.path.isfile(path_bin): - # Was downloaded and extracted by CM + # Was downloaded and extracted by MLC env['MLC_RCLONE_BIN_WITH_PATH'] = path_bin env['+PATH'] = [cur_dir] diff --git a/script/get-rclone/meta.yaml b/script/get-rclone/meta.yaml index b0727d484..75403386a 100644 --- a/script/get-rclone/meta.yaml +++ b/script/get-rclone/meta.yaml @@ -27,4 +27,4 @@ variations: env: MLC_RCLONE_SYSTEM: 'yes' warnings: - - This CM script will install rclone using sudo/brew! + - This MLC script will install rclone using sudo/brew! diff --git a/script/get-sys-utils-cm/do_pip_installs.sh.old b/script/get-sys-utils-cm/do_pip_installs.sh.old deleted file mode 100644 index 441f884dc..000000000 --- a/script/get-sys-utils-cm/do_pip_installs.sh.old +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -PIP_EXTRA=`python3 -c "import pkg_resources; print(' --break-system-packages ' if int(pkg_resources.get_distribution('pip').version.split('.')[0]) >= 23 else '')"` -cmd="python3 -m pip install -r ${MLC_TMP_CURRENT_SCRIPT_PATH}/requirements.txt ${MLC_PYTHON_PIP_USER} ${MLC_PYTHON_PIP_COMMON_EXTRA} ${PIP_EXTRA}" -echo $cmd -eval $cmd diff --git a/script/get-sys-utils-cm/COPYRIGHT.md b/script/get-sys-utils-mlc/COPYRIGHT.md similarity index 100% rename from script/get-sys-utils-cm/COPYRIGHT.md rename to script/get-sys-utils-mlc/COPYRIGHT.md diff --git a/script/get-sys-utils-cm/customize.py b/script/get-sys-utils-mlc/customize.py similarity index 99% rename from script/get-sys-utils-cm/customize.py rename to script/get-sys-utils-mlc/customize.py index ed72dac02..7a5c4ff83 100644 --- a/script/get-sys-utils-cm/customize.py +++ b/script/get-sys-utils-mlc/customize.py @@ -87,7 +87,7 @@ def preprocess(i): logger.info( '***********************************************************************') logger.info( - 'This script will attempt to install minimal system dependencies for CM.') + 'This script will attempt to install minimal system dependencies for MLC.') logger.info('Note that you may be asked for your SUDO password ...') logger.info( '***********************************************************************') diff --git a/script/get-sys-utils-cm/do_pip_installs.sh b/script/get-sys-utils-mlc/do_pip_installs.sh similarity index 100% rename from script/get-sys-utils-cm/do_pip_installs.sh rename to script/get-sys-utils-mlc/do_pip_installs.sh diff --git a/script/get-sys-utils-cm/meta.yaml b/script/get-sys-utils-mlc/meta.yaml similarity index 95% rename from script/get-sys-utils-cm/meta.yaml rename to script/get-sys-utils-mlc/meta.yaml index 83e39e9a4..dfce36f85 100644 --- a/script/get-sys-utils-cm/meta.yaml +++ b/script/get-sys-utils-mlc/meta.yaml @@ -1,4 +1,4 @@ -alias: get-sys-utils-cm +alias: get-sys-utils-mlc uid: bc90993277e84b8e automation_alias: script diff --git a/script/get-sys-utils-cm/requirements.txt b/script/get-sys-utils-mlc/requirements.txt similarity index 100% rename from script/get-sys-utils-cm/requirements.txt rename to script/get-sys-utils-mlc/requirements.txt diff --git a/script/get-sys-utils-cm/run-arch.sh b/script/get-sys-utils-mlc/run-arch.sh similarity index 100% rename from script/get-sys-utils-cm/run-arch.sh rename to script/get-sys-utils-mlc/run-arch.sh diff --git a/script/get-sys-utils-cm/run-debian.sh b/script/get-sys-utils-mlc/run-debian.sh similarity index 100% rename from script/get-sys-utils-cm/run-debian.sh rename to script/get-sys-utils-mlc/run-debian.sh diff --git a/script/get-sys-utils-cm/run-macos.sh b/script/get-sys-utils-mlc/run-macos.sh similarity index 100% rename from script/get-sys-utils-cm/run-macos.sh rename to script/get-sys-utils-mlc/run-macos.sh diff --git a/script/get-sys-utils-cm/run-rhel.sh b/script/get-sys-utils-mlc/run-rhel.sh similarity index 96% rename from script/get-sys-utils-cm/run-rhel.sh rename to script/get-sys-utils-mlc/run-rhel.sh index e91d70d07..0d61d5ccb 100644 --- a/script/get-sys-utils-cm/run-rhel.sh +++ b/script/get-sys-utils-mlc/run-rhel.sh @@ -24,7 +24,8 @@ ${MLC_SUDO} ${MLC_PACKAGE_TOOL} update && \ ca-certificates curl cmake \ gcc git g++ \ libtool libffi-devel libssl-devel\ - zlib-devel \ + procps-ng \ + zlib-devel \ libbz2-devel \ openssh-client \ make mesa-libGL \ diff --git a/script/get-sys-utils-cm/run-sles.sh b/script/get-sys-utils-mlc/run-sles.sh similarity index 100% rename from script/get-sys-utils-cm/run-sles.sh rename to script/get-sys-utils-mlc/run-sles.sh diff --git a/script/get-sys-utils-cm/run-ubuntu.sh b/script/get-sys-utils-mlc/run-ubuntu.sh similarity index 100% rename from script/get-sys-utils-cm/run-ubuntu.sh rename to script/get-sys-utils-mlc/run-ubuntu.sh diff --git a/script/get-xilinx-sdk/run.sh b/script/get-xilinx-sdk/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/get-xilinx-sdk/run.sh +++ b/script/get-xilinx-sdk/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/install-mlperf-logging-from-src/run.sh b/script/install-mlperf-logging-from-src/run.sh index 0d5d73c4c..0aa4de475 100644 --- a/script/install-mlperf-logging-from-src/run.sh +++ b/script/install-mlperf-logging-from-src/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/install-nccl-libs/run.sh b/script/install-nccl-libs/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/install-nccl-libs/run.sh +++ b/script/install-nccl-libs/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/install-numactl-from-src/meta.yaml b/script/install-numactl-from-src/meta.yaml index a03250cdf..0da55b589 100644 --- a/script/install-numactl-from-src/meta.yaml +++ b/script/install-numactl-from-src/meta.yaml @@ -58,4 +58,4 @@ variations: env: MLC_GIT_CHECKOUT_TAG: '#' warnings: -- This CM script will need sudo to install numactl! +- This MLC script will need sudo to install numactl! diff --git a/script/prepare-training-data-bert/run-nvidia.sh b/script/prepare-training-data-bert/run-nvidia.sh index 016f6e07d..ad17be414 100644 --- a/script/prepare-training-data-bert/run-nvidia.sh +++ b/script/prepare-training-data-bert/run-nvidia.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/prepare-training-data-bert/run-reference.sh b/script/prepare-training-data-bert/run-reference.sh index a72f3adfd..543114a5b 100644 --- a/script/prepare-training-data-bert/run-reference.sh +++ b/script/prepare-training-data-bert/run-reference.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/prepare-training-data-bert/run.sh b/script/prepare-training-data-bert/run.sh index 194b53e80..30655574c 100644 --- a/script/prepare-training-data-bert/run.sh +++ b/script/prepare-training-data-bert/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/prepare-training-data-resnet/run-nvidia.sh b/script/prepare-training-data-resnet/run-nvidia.sh index 18fa70634..a78820a82 100644 --- a/script/prepare-training-data-resnet/run-nvidia.sh +++ b/script/prepare-training-data-resnet/run-nvidia.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/prepare-training-data-resnet/run-reference.sh b/script/prepare-training-data-resnet/run-reference.sh index a2f4a5eda..cdb890d58 100644 --- a/script/prepare-training-data-resnet/run-reference.sh +++ b/script/prepare-training-data-resnet/run-reference.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/prune-bert-models/customize.py b/script/prune-bert-models/customize.py index e69cde689..8a36d1664 100644 --- a/script/prune-bert-models/customize.py +++ b/script/prune-bert-models/customize.py @@ -33,7 +33,7 @@ def preprocess(i): logger.info('') print( - 'Local CM cache path to the updated BERT pruner src from NeurIPS 2022: ' + + 'Local MLC cache path to the updated BERT pruner src from NeurIPS 2022: ' + env['MLC_GIT_REPO_BERT_PRUNER_NEURIPS_2022_CHECKOUT_PATH']) logger.info('') diff --git a/script/reproduce-mlperf-octoml-tinyml-results/meta.yaml b/script/reproduce-mlperf-octoml-tinyml-results/meta.yaml index 055bb03dd..d04ad19f5 100644 --- a/script/reproduce-mlperf-octoml-tinyml-results/meta.yaml +++ b/script/reproduce-mlperf-octoml-tinyml-results/meta.yaml @@ -7,7 +7,7 @@ default_version: r1.0 deps: - tags: detect,os - tags: detect,cpu -- tags: get,sys-utils-cm +- tags: get,sys-utils-mlc - names: - python3 - python diff --git a/script/reproduce-mlperf-training-nvidia/meta.yaml b/script/reproduce-mlperf-training-nvidia/meta.yaml index 263ab9d63..cb2c8e926 100644 --- a/script/reproduce-mlperf-training-nvidia/meta.yaml +++ b/script/reproduce-mlperf-training-nvidia/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: reproduce-mlperf-training-nvidia uid: f183628f292341e2 cache: false @@ -9,7 +9,7 @@ automation_uid: 5b4e0237da074764 category: "Reproduce MLPerf benchmarks" -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - reproduce - mlcommons @@ -29,7 +29,7 @@ new_state_keys: - mlperf-training-implementation - MLC_SUT_* -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: diff --git a/script/run-all-mlperf-models/run-bert-macos.sh b/script/run-all-mlperf-models/run-bert-macos.sh index bc7a75747..9b103d0c9 100644 --- a/script/run-all-mlperf-models/run-bert-macos.sh +++ b/script/run-all-mlperf-models/run-bert-macos.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-bert.sh b/script/run-all-mlperf-models/run-bert.sh index 815e144ec..f3ffa1222 100644 --- a/script/run-all-mlperf-models/run-bert.sh +++ b/script/run-all-mlperf-models/run-bert.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-cpp-implementation.sh b/script/run-all-mlperf-models/run-cpp-implementation.sh index 3d4c7b93d..0e974b3d9 100644 --- a/script/run-all-mlperf-models/run-cpp-implementation.sh +++ b/script/run-all-mlperf-models/run-cpp-implementation.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-mobilenet-models.sh b/script/run-all-mlperf-models/run-mobilenet-models.sh index 849bb48ec..557332d93 100644 --- a/script/run-all-mlperf-models/run-mobilenet-models.sh +++ b/script/run-all-mlperf-models/run-mobilenet-models.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-nvidia-4090.sh b/script/run-all-mlperf-models/run-nvidia-4090.sh index 135a270d0..05184c117 100644 --- a/script/run-all-mlperf-models/run-nvidia-4090.sh +++ b/script/run-all-mlperf-models/run-nvidia-4090.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-nvidia-a100.sh b/script/run-all-mlperf-models/run-nvidia-a100.sh index a3489e7d2..715d28077 100644 --- a/script/run-all-mlperf-models/run-nvidia-a100.sh +++ b/script/run-all-mlperf-models/run-nvidia-a100.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-nvidia-t4.sh b/script/run-all-mlperf-models/run-nvidia-t4.sh index adde34344..adb1eef9e 100644 --- a/script/run-all-mlperf-models/run-nvidia-t4.sh +++ b/script/run-all-mlperf-models/run-nvidia-t4.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-reference-models.sh b/script/run-all-mlperf-models/run-reference-models.sh index 01766158a..2a0948892 100644 --- a/script/run-all-mlperf-models/run-reference-models.sh +++ b/script/run-all-mlperf-models/run-reference-models.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-resnet50-macos.sh b/script/run-all-mlperf-models/run-resnet50-macos.sh index 5b83b9a9b..9b49434f8 100644 --- a/script/run-all-mlperf-models/run-resnet50-macos.sh +++ b/script/run-all-mlperf-models/run-resnet50-macos.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-resnet50.sh b/script/run-all-mlperf-models/run-resnet50.sh index a2144bf0a..797f174f5 100644 --- a/script/run-all-mlperf-models/run-resnet50.sh +++ b/script/run-all-mlperf-models/run-resnet50.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/run-retinanet-sh b/script/run-all-mlperf-models/run-retinanet-sh index 3f10a88ee..abf29a632 100644 --- a/script/run-all-mlperf-models/run-retinanet-sh +++ b/script/run-all-mlperf-models/run-retinanet-sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-all-mlperf-models/template.sh b/script/run-all-mlperf-models/template.sh index 9a5fb1893..f0359ced4 100644 --- a/script/run-all-mlperf-models/template.sh +++ b/script/run-all-mlperf-models/template.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 62eb7f535..04d7e1f92 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -33,7 +33,7 @@ def preprocess(i): 'tags': env['MLC_DOCKER_RUN_SCRIPT_TAGS']}) if len(r['list']) < 1: raise Exception( - 'CM script with tags ' + + 'MLC script with tags ' + env['MLC_DOCKER_RUN_SCRIPT_TAGS'] + ' not found!') diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 8cf0fbe1c..a0c82e593 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -331,9 +331,9 @@ def preprocess(i): if state.get("mlc-mlperf-inference-results"): # print(state["mlc-mlperf-inference-results"]) for sut in state["mlc-mlperf-inference-results"]: # only one sut will be there - # Better to do this in a stand alone CM script with proper deps but + # Better to do this in a stand alone MLC script with proper deps but # currently we manage this by modifying the sys path of the python - # executing CM + # executing MLC from tabulate import tabulate # noqa logger.info(f"{sut}") diff --git a/script/runtime-system-infos/meta.yaml b/script/runtime-system-infos/meta.yaml index 3e3186c4b..37c383814 100644 --- a/script/runtime-system-infos/meta.yaml +++ b/script/runtime-system-infos/meta.yaml @@ -1,4 +1,4 @@ -# Identification of this CM script +# Identification of this MLC script alias: runtime-system-infos uid: 755cf27627784001 cache: false @@ -9,7 +9,7 @@ category: "MLPerf benchmark support" docker: real_run: False -# User-friendly tags to find this CM script +# User-friendly tags to find this MLC script tags: - runtime - system @@ -21,7 +21,7 @@ input_mapping: interval: MLC_SYSTEM_INFO_MEASUREMENT_INTERVAL -# Dependencies on other CM scripts +# Dependencies on other MLC scripts deps: @@ -32,7 +32,7 @@ deps: - tags: detect,cpu # Install system dependencies on a given host - - tags: get,sys-utils-cm + - tags: get,sys-utils-mlc # Detect python3 - tags: get,python3 diff --git a/script/set-device-settings-qaic/run.sh b/script/set-device-settings-qaic/run.sh index dab1a87d4..2220d36c6 100644 --- a/script/set-device-settings-qaic/run.sh +++ b/script/set-device-settings-qaic/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/set-performance-mode/run-ubuntu.sh b/script/set-performance-mode/run-ubuntu.sh index 71b8ce802..c5889cfa9 100644 --- a/script/set-performance-mode/run-ubuntu.sh +++ b/script/set-performance-mode/run-ubuntu.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/set-performance-mode/run.sh b/script/set-performance-mode/run.sh index 821adb3f9..ee74b85b6 100644 --- a/script/set-performance-mode/run.sh +++ b/script/set-performance-mode/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/set-user-limits/run.sh b/script/set-user-limits/run.sh index 32cf4d51e..4a22ce805 100644 --- a/script/set-user-limits/run.sh +++ b/script/set-user-limits/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -#CM Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} +#MLC Script location: ${MLC_TMP_CURRENT_SCRIPT_PATH} #To export any variable #echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out diff --git a/script/set-venv/customize.py b/script/set-venv/customize.py index daf23ea4f..64b6d3607 100644 --- a/script/set-venv/customize.py +++ b/script/set-venv/customize.py @@ -80,10 +80,10 @@ def preprocess(i): shell = shell.replace('MLC_SET_VENV_WORK', 'work') if shell == '': shell = 'cmd' - cmd = 'cd {} & call {} & set MLC_REPOS=%CD%\\{}\\CM & {}\n'.format( + cmd = 'cd {} & call {} & set MLC_REPOS=%CD%\\{}\\MLC & {}\n'.format( name, activate_script, name, shell) else: - cmd = '#!/bin/bash\n\ncd {} ; source {} ; export MLC_REPOS=$PWD/CM ; cd work\n'.format( + cmd = '#!/bin/bash\n\ncd {} ; source {} ; export MLC_REPOS=$PWD/MLC ; cd work\n'.format( name, activate_script) with open(script_file, 'w') as f: From ec30158a28a7a0fe92749cbf27081b8b7d179cc8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 23 Jul 2025 22:04:16 +0000 Subject: [PATCH 029/124] [Automated Commit] Document script/get-cache-dir/meta.yaml [skip ci] --- script/get-cache-dir/README.md | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 script/get-cache-dir/README.md diff --git a/script/get-cache-dir/README.md b/script/get-cache-dir/README.md new file mode 100644 index 000000000..172b85eec --- /dev/null +++ b/script/get-cache-dir/README.md @@ -0,0 +1,62 @@ +# README for get-cache-dir +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. + +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +``` +mkdir /mnt/user/MLC +ln -s /mnt/user/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. +## Run Commands + +```bash +mlcr get,cache,dir,directory +``` + +No script specific inputs +### Generic Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--input` | Input to the script passed using the env key `MLC_INPUT` | | `` | +| `--output` | Output from the script passed using the env key `MLC_OUTPUT` | | `` | +| `--outdirname` | The directory to store the script output | | `cache directory ($HOME/MLC/repos/local/cache/<>) if the script is cacheable or else the current directory` | +| `--outbasename` | The output file/folder name | | `` | +| `--name` | | | `` | +| `--extra_cache_tags` | Extra cache tags to be added to the cached entry when the script results are saved | | `` | +| `--skip_compile` | Skip compilation | | `False` | +| `--skip_run` | Skip run | | `False` | +| `--accept_license` | Accept the required license requirement to run the script | | `False` | +| `--skip_system_deps` | Skip installing any system dependencies | | `False` | +| `--git_ssh` | Use SSH for git repos | | `False` | +| `--gh_token` | Github Token | | `` | +| `--hf_token` | Huggingface Token | | `` | +| `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Ungrouped + +- `name.#` _(# can be substituted dynamically)_ From bf2f7f06c4a8961848d24ddaaf4ca174f5a87016 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Jul 2025 18:09:34 +0100 Subject: [PATCH 030/124] Update document-scripts.yml: don't commit unless there are changes to READMEs (#537) --- .github/workflows/document-scripts.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/document-scripts.yml b/.github/workflows/document-scripts.yml index 9298efbf9..53b621548 100644 --- a/.github/workflows/document-scripts.yml +++ b/.github/workflows/document-scripts.yml @@ -76,8 +76,7 @@ jobs: git config --global user.name github-actions[bot] git config --global user.email "github-actions[bot]@users.noreply.github.com" # Commit changes - git commit -m '[Automated Commit] Document ${{ matrix.modified_metas.file}} [skip ci]' - git push + git diff-index --quiet HEAD || (git commit -am "[Automated Commit] Document ${{ matrix.modified_metas.file}} [skip ci]" ) && git push) From aa3d831347864b1e7bc1282669a321bbe8f5df8b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 24 Jul 2025 22:36:11 +0100 Subject: [PATCH 031/124] Improve docs template (#538) * Improve docs template --- automation/script/doc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automation/script/doc.py b/automation/script/doc.py index 7c8fb6538..340d3805c 100644 --- a/automation/script/doc.py +++ b/automation/script/doc.py @@ -76,10 +76,10 @@ def get_setup_readme(script_repo): if '@' in repo_name: repo_name = repo_name.split('@')[1] - setup_readme = f"""`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do + setup_readme = f"""`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. From 6c4cff5b0f8a54f018aabbdc29b7051d838018c8 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Jul 2025 15:57:55 +0530 Subject: [PATCH 032/124] Fix for whisper reference benchmark (#540) * pin vllm branch * change from branch to sha to get full history --- .../app-mlperf-inference-mlcommons-python/customize.py | 2 +- script/app-mlperf-inference-mlcommons-python/meta.yaml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index c048e8f3c..c29dd26d2 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -580,7 +580,7 @@ def get_run_cmd_reference( --dataset_dir {x}{env['MLC_DATASET_WHISPER_PATH']}{x} \ --manifest {x}{os.path.join(env['MLC_DATASET_WHISPER_PATH'], "data", "dev-all-repack.json")}{x} \ --log_dir {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \ - --model-path {x}{env['MLC_ML_MODEL_WHISPER_PATH']}{x} \ + --model_path {x}{env['MLC_ML_MODEL_WHISPER_PATH']}{x} \ {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ {scenario_extra_options} {mode_extra_options}""" diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index c5ebee492..7dab92c44 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -211,6 +211,8 @@ deps: - cpu - tags: install-vllm-from-src,_cpu + names: + - vllm-from-src enable_if_env: MLC_MLPERF_BACKEND: - vllm @@ -1492,6 +1494,9 @@ variations: group: models env: MLC_MODEL: whisper + adr: + vllm-from-src: + tags: _sha.b124e1085b1bf977e3dac96d99ffd9d8ddfdb6cc deps: - tags: get,generic-python-lib,_package.py-libnuma - tags: get,generic-sys-util,_libnuma-dev @@ -1516,10 +1521,6 @@ variations: MLC_RUN_STATE_DOCKER: - "yes" - whisper,cpu: - deps: - - tags: install-vllm-from-src,_cpu - deepseek-r1: group: models env: From b3589744f16dbde1dd6b52e25736d24c568e098c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Jul 2025 15:58:54 +0530 Subject: [PATCH 033/124] Add script to detect/install apptainer (#536) * add script to detect/install apptainer --- script/get-apptainer/COPYRIGHT.md | 9 +++ script/get-apptainer/customize.py | 85 ++++++++++++++++++++++++++ script/get-apptainer/install-centos.sh | 8 +++ script/get-apptainer/install-ubuntu.sh | 12 ++++ script/get-apptainer/install.bat | 2 + script/get-apptainer/install.sh | 2 + script/get-apptainer/meta.yaml | 22 +++++++ script/get-apptainer/run.bat | 3 + script/get-apptainer/run.sh | 3 + 9 files changed, 146 insertions(+) create mode 100644 script/get-apptainer/COPYRIGHT.md create mode 100644 script/get-apptainer/customize.py create mode 100644 script/get-apptainer/install-centos.sh create mode 100644 script/get-apptainer/install-ubuntu.sh create mode 100644 script/get-apptainer/install.bat create mode 100644 script/get-apptainer/install.sh create mode 100644 script/get-apptainer/meta.yaml create mode 100644 script/get-apptainer/run.bat create mode 100644 script/get-apptainer/run.sh diff --git a/script/get-apptainer/COPYRIGHT.md b/script/get-apptainer/COPYRIGHT.md new file mode 100644 index 000000000..2d6a2775e --- /dev/null +++ b/script/get-apptainer/COPYRIGHT.md @@ -0,0 +1,9 @@ +# Copyright Notice + +ยฉ 2023-2025 MLCommons. All Rights Reserved. + +This file is licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License can be obtained at: + +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is provided on an "AS IS" basis, without warranties or conditions of any kind, either express or implied. Please refer to the License for the specific language governing permissions and limitations under the License. diff --git a/script/get-apptainer/customize.py b/script/get-apptainer/customize.py new file mode 100644 index 000000000..6750eede3 --- /dev/null +++ b/script/get-apptainer/customize.py @@ -0,0 +1,85 @@ +from mlc import utils +import os + + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + automation = i['automation'] + + recursion_spaces = i['recursion_spaces'] + + file_name_apptainer = 'apptainer.exe' if os_info['platform'] == 'windows' else 'apptainer' + + if 'MLC_APPTAINER_BIN_WITH_PATH' not in env: + env['FILE_NAME'] = file_name_apptainer + env['CONTAINER_TOOL_NAME'] = "apptainer" + r = i['automation'].find_artifact({'file_name': file_name_apptainer, + 'env': env, + 'os_info': os_info, + 'default_path_env_key': 'PATH', + 'detect_version': True, + 'env_path_key': 'MLC_APPTAINER_BIN_WITH_PATH', + 'run_script_input': i['run_script_input'], + 'recursion_spaces': recursion_spaces}) + if r['return'] > 0: + if r['return'] == 16: + run_file_name = "install" + r = automation.run_native_script( + {'run_script_input': i['run_script_input'], 'env': env, 'script_name': run_file_name}) + if r['return'] > 0: + return r + else: + return r + + return {'return': 0} + + +def detect_version(i): + r = i['automation'].parse_version({ + 'match_text': r'[Aa]pptainer version\s*([\d.]+)', + 'group_number': 1, + 'env_key': 'MLC_APPTAINER_VERSION', + 'which_env': i['env'] + }) + if r['return'] > 0: + return r + + logger = i['automation'].logger + + version = r['version'] + + tool = "apptainer" + + logger.info( + i['recursion_spaces'] + + ' Detected version: {}'.format(version)) + return {'return': 0, 'version': version, "tool": tool} + + +def postprocess(i): + env = i['env'] + + r = detect_version(i) + + if r['return'] > 0: + return r + + version = r['version'] + tool = r['tool'] + found_file_path = env['MLC_APPTAINER_BIN_WITH_PATH'] + + found_path = os.path.dirname(found_file_path) + env['MLC_APPTAINER_INSTALLED_PATH'] = found_path + env['+PATH'] = [found_path] + + env['MLC_APPTAINER_CACHE_TAGS'] = 'version-' + version + + env['MLC_APPTAINER_VERSION'] = version + + env['MLC_CONTAINER_TOOL'] = tool + + return {'return': 0, 'version': version} diff --git a/script/get-apptainer/install-centos.sh b/script/get-apptainer/install-centos.sh new file mode 100644 index 000000000..be9879bf2 --- /dev/null +++ b/script/get-apptainer/install-centos.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +${MLC_SUDO} yum install -y epel-release +${MLC_SUDO} yum install -y apptainer +test $? -eq 0 || exit $? + +# Print version +apptainer --version diff --git a/script/get-apptainer/install-ubuntu.sh b/script/get-apptainer/install-ubuntu.sh new file mode 100644 index 000000000..8a7036669 --- /dev/null +++ b/script/get-apptainer/install-ubuntu.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +${MLC_SUDO} apt-get update +${MLC_SUDO} apt-get install -y software-properties-common +${MLC_SUDO} add-apt-repository -y ppa:apptainer/ppa +${MLC_SUDO} apt-get update +${MLC_SUDO} apt-get install -y apptainer +test $? -eq 0 || exit $? + +# Print version +apptainer --version + diff --git a/script/get-apptainer/install.bat b/script/get-apptainer/install.bat new file mode 100644 index 000000000..dfde606b6 --- /dev/null +++ b/script/get-apptainer/install.bat @@ -0,0 +1,2 @@ +echo "Please install apptainer to continue" +exit 1 diff --git a/script/get-apptainer/install.sh b/script/get-apptainer/install.sh new file mode 100644 index 000000000..dfde606b6 --- /dev/null +++ b/script/get-apptainer/install.sh @@ -0,0 +1,2 @@ +echo "Please install apptainer to continue" +exit 1 diff --git a/script/get-apptainer/meta.yaml b/script/get-apptainer/meta.yaml new file mode 100644 index 000000000..39743ec39 --- /dev/null +++ b/script/get-apptainer/meta.yaml @@ -0,0 +1,22 @@ +alias: get-apptainer +automation_alias: script +automation_uid: 5b4e0237da074764 +cache: true +category: Detection or installation of tools and artifacts +deps: +- tags: detect,os +- tags: detect,sudo +input_description: {} +input_mapping: {} +new_env_keys: +- MLC_APPTAINER_VERSION +- MLC_CONTAINER_TOOL +new_state_keys: [] +post_deps: [] +posthook_deps: [] +prehook_deps: [] +tags: +- get-apptainer +uid: 37c5bc01c11f49fa +variations: {} +versions: {} diff --git a/script/get-apptainer/run.bat b/script/get-apptainer/run.bat new file mode 100644 index 000000000..a90c05a64 --- /dev/null +++ b/script/get-apptainer/run.bat @@ -0,0 +1,3 @@ +@echo off +apptainer --version > tmp-ver.out +if %errorlevel% neq 0 exit /b 1 diff --git a/script/get-apptainer/run.sh b/script/get-apptainer/run.sh new file mode 100644 index 000000000..38a153a65 --- /dev/null +++ b/script/get-apptainer/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash +apptainer --version > tmp-ver.out +test $? -eq 0 || exit 1 From 9bd7ce423fdf721d9e8e5835725e406ec601074e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Jul 2025 15:59:52 +0530 Subject: [PATCH 034/124] Output the submission directory in push to GitHub (#533) --- .../customize.py | 12 ++++++++++++ .../meta.yaml | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/script/push-mlperf-inference-results-to-github/customize.py b/script/push-mlperf-inference-results-to-github/customize.py index 9b2df4d0f..b11bbd63a 100644 --- a/script/push-mlperf-inference-results-to-github/customize.py +++ b/script/push-mlperf-inference-results-to-github/customize.py @@ -42,3 +42,15 @@ def preprocess(i): env['MLC_GIT_PUSH_CMD'] = "git push" return {'return': 0} + + +def postprocess(i): + + os_info = i['os_info'] + env = i['env'] + meta = i['meta'] + automation = i['automation'] + + env['MLC_MLPERF_RESULTS_SYNC_DIR'] = env['MLC_GIT_REPO_CHECKOUT_PATH'] + + return {'return': 0} diff --git a/script/push-mlperf-inference-results-to-github/meta.yaml b/script/push-mlperf-inference-results-to-github/meta.yaml index 2e92bf7c8..3535b31a3 100644 --- a/script/push-mlperf-inference-results-to-github/meta.yaml +++ b/script/push-mlperf-inference-results-to-github/meta.yaml @@ -4,6 +4,10 @@ automation_uid: 5b4e0237da074764 category: MLPerf benchmark support default_env: MLC_MLPERF_RESULTS_GIT_REPO_URL: https://github.com/mlcommons/mlperf_inference_submissions_v4.0 +new_env_keys: + - MLC_MLPERF_RESULTS_SYNC_DIR +print_env_at_the_end: + MLC_MLPERF_RESULTS_SYNC_DIR: Path to the locally synced submission directory deps: - names: - python3 From 443c7c15344fc00dec54e6105c0536203bb0aa60 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 25 Jul 2025 22:05:59 +0530 Subject: [PATCH 035/124] Modifications for llama3.1-8b edge (#539) --- .../meta.yaml | 41 +++++----- script/app-mlperf-inference/meta.yaml | 74 ++++++++++++------- script/get-dataset-cnndm/meta.yaml | 4 +- 3 files changed, 73 insertions(+), 46 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 7dab92c44..197d8ec94 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -1449,10 +1449,9 @@ variations: env: MLC_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<>>+cpu.html" - llama3_1-8b: - group: models - env: - MLC_MODEL: llama3_1-8b + llama3_1-8b_: + group: + models deps: - tags: get,generic-python-lib,_package.transformers - tags: get,generic-python-lib,_package.sentencepiece @@ -1469,27 +1468,35 @@ variations: names: - llama3_1-8b - llama3-8b - ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader - names: - - cnndm-llama3-edge - enable_if_env: - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - edge - skip_if_env: - MLC_RUN_STATE_DOCKER: - - "yes" + + llama3_1-8b: + base: + - llama3_1-8b_ + env: + MLC_MODEL: llama3_1-8b + deps: ## CNNDM for Llama3 8B model - datacenter - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader names: - cnndm-llama3-datacenter - enable_if_env: - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - datacenter skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" + llama3_1-8b-edge: + base: + - llama3_1-8b_ + env: + MLC_MODEL: llama3_1-8b-edge + deps: + ## CNNDM for Llama3 8B model - edge + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader + names: + - cnndm-llama3-edge + skip_if_env: + MLC_RUN_STATE_DOCKER: + - "yes" + whisper: group: models env: diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index b1d7ef38e..16be0570f 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -955,10 +955,31 @@ variations: - mlperf-accuracy-script - waymo-accuracy-script tags: run,accuracy,mlperf,_waymo - - llama3_1-8b: + + llama3_1-8b_: group: model + adr: + numpy: + version_max: "1.26.999" + version_max_usable: "1.26.4" + docker: + base_image: ubuntu:24.04 + mounts: + - "${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}" + - "${{ MLC_DATASET_CNNDM_EVAL_PATH }}:${{ MLC_DATASET_CNNDM_EVAL_PATH }}" + deps: + - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader + enable_if_env: + MLC_USE_ML_MODEL_FROM_HOST: + - 'yes' + names: + - llama3_1-8b + - llama3-8b + + llama3_1-8b: + base: + - llama3_1-8b_ add_deps_recursive: mlperf-inference-implementation: tags: _llama3_1-8b @@ -972,42 +993,50 @@ variations: - all MLC_MLPERF_ACCURACY_RESULTS_DIR: - 'on' - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - edge skip_if_env: MLC_MLPERF_IMPLEMENTATION: - nvidia names: - mlperf-accuracy-script - llama3_1-8b-accuracy-script - tags: run,accuracy,mlperf,_cnndm_llama_3,_edge + tags: run,accuracy,mlperf,_cnndm_llama_3,_datacenter + docker: + deps: + ## CNNDM for Llama3 8B model - datacenter + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader + names: + - cnndm-llama3-datacenter + enable_if_env: + MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: + - datacenter + MLC_USE_DATASET_FROM_HOST: + - 'yes' + + llama3.1-8b-edge: + base: + - llama3_1-8b_ + add_deps_recursive: + mlperf-inference-implementation: + tags: _llama3_1-8b-edge + env: + MLC_MODEL: + llama3_1-8b-edge + posthook_deps: - enable_if_env: MLC_MLPERF_LOADGEN_MODE: - accuracy - all MLC_MLPERF_ACCURACY_RESULTS_DIR: - 'on' - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - datacenter skip_if_env: MLC_MLPERF_IMPLEMENTATION: - nvidia names: - mlperf-accuracy-script - llama3_1-8b-accuracy-script - tags: run,accuracy,mlperf,_cnndm_llama_3,_datacenter + tags: run,accuracy,mlperf,_cnndm_llama_3,_edge docker: - mounts: - - "${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}" - - "${{ MLC_DATASET_CNNDM_EVAL_PATH }}:${{ MLC_DATASET_CNNDM_EVAL_PATH }}" deps: - - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader - enable_if_env: - MLC_USE_ML_MODEL_FROM_HOST: - - 'yes' - names: - - llama3_1-8b - - llama3-8b ## CNNDM for Llama3 8B model - edge - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader names: @@ -1017,15 +1046,6 @@ variations: - edge MLC_USE_DATASET_FROM_HOST: - 'yes' - ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader - names: - - cnndm-llama3-datacenter - enable_if_env: - MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: - - datacenter - MLC_USE_DATASET_FROM_HOST: - - 'yes' whisper: group: diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index 1e891215c..4d38491f8 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -153,7 +153,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH - validation,datacenter,llama3,mlc,r2_downlaoder: + validation,datacenter,llama3,mlc,r2_downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,datacenter @@ -188,4 +188,4 @@ tests: - validation,edge,r2_downloader,llama3,mlc,dry-run - validation,datacenter,r2_downloader,llama3,mlc,dry-run - calibration,r2_downloader,llama3,mlc,dry-run - # - calibration,rclone,llama3,mlc,dry-run \ No newline at end of file + # - calibration,rclone,llama3,mlc,dry-run From 84de37a4b2ec613ff44b731c7d08b22c6d3ae0e0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 26 Jul 2025 12:47:21 +0100 Subject: [PATCH 036/124] Add initial draft of help.py (#542) * Added initial version of help.py --- automation/script/help.py | 182 ++++++++++++++++++++++++++++++++++++ automation/script/module.py | 39 ++++++-- 2 files changed, 213 insertions(+), 8 deletions(-) create mode 100644 automation/script/help.py diff --git a/automation/script/help.py b/automation/script/help.py new file mode 100644 index 000000000..c62fc790b --- /dev/null +++ b/automation/script/help.py @@ -0,0 +1,182 @@ +import os +from mlc import utils +from utils import * +import os +from collections import defaultdict + + +def display_help(self_module, input_params): + """ + Generates the documentation of MLC scripts. + + Args: + self_module: Reference to the current module for internal calls. + input_params: Dictionary containing input parameters. + + Returns: + Dictionary with the result of the operation. Keys: + - 'return': 0 on success, >0 on error. + - 'error': Error message (if any). + """ + + # Extract and handle basic inputs + quiet = input_params.get('quiet', False) + logger = self_module.logger + env = input_params.get('env', {}) + generic_inputs = self_module.input_flags_converted_to_env + + # Step 2: Search for scripts + search_result = self_module.search(input_params.copy()) + if search_result['return'] > 0: + return search_result + + scripts_list = search_result['list'] + if not scripts_list: + return {'return': 1, 'error': 'No scripts were found'} + + # Step 4: Iterate over scripts and generate help output + for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): + metadata = script.meta + print_script_help(metadata) + + return {'return': 0} + + +def print_script_help(metadata): + print(f"Script Name: {metadata.get('alias', metadata['uid'])}") + print(f"Tags: {', '.join(metadata.get('tags', []))}") + # Print the run commands + print_run_commands(metadata) + print("") + print("Script Inputs:") + print("") + + input_mapping = metadata.get('input_mapping', {}) + input_description = metadata.get('input_description', {}) + default_env = metadata.get('default_env', {}) + + reverse_map = defaultdict(list) + for k, v in input_mapping.items(): + reverse_map[v].append(k) + + if input_description: + for i in input_description: + if i in input_mapping and input_mapping[i] in default_env: + input_description[i]['default'] = default_env[input_mapping[i]] + + # Add alias entries + for mapped_env, keys in reverse_map.items(): + if len(keys) > 1: + canonical = keys[0] + for alias in keys[1:]: + if alias in input_description: + input_description[alias] = {} + input_description[alias]['alias'] = canonical + input_description[alias]['desc'] = f"""Alias for {canonical}""" + + for key, value in input_mapping.items(): + desc = input_description.get(key, {}).get( + 'desc', 'No description available') + default = input_description.get(key, {}).get('default', 'None') + # Use .ljust(15) to ensure the key occupies 15 characters minimum + print( + f" --{key.ljust(26)} : maps to --env.{value}\n{' '.ljust(35)}{desc}\n{' '.ljust(35)}Default: {default}\n") + + variations = metadata.get('variations', {}) + + if variations: + print_variations_help(variations) + else: + print(" - No variations.") + + print("\n" + "=" * 40 + "\n") # Separator for clarity + + +def print_variations_help(variations): + # Data structures + aliases = {} + alias_reverse = defaultdict(list) + bases = defaultdict(list) + variation_groups = {} + main_variations = {} + + # First pass: classify and build maps + for name, attrs in variations.items(): + if "," in name: + continue # Skip composite variations + if not isinstance(attrs, dict): + main_variations[name] = {} + continue + if "alias" in attrs: + aliases[name] = attrs["alias"] + alias_reverse[attrs["alias"]].append(name) + else: + main_variations[name] = attrs + # Group + group = attrs.get("group", "ungrouped") + if isinstance(group, list): + group = group[0] if group else "ungrouped" + variation_groups[name] = group + # Base + base = attrs.get("base", []) + if isinstance(base, str): + base = [base] + bases[name] = base + + # Build grouped output in a simpler format + grouped_output = defaultdict(list) + + for var in sorted(main_variations.keys()): + group = variation_groups.get(var, "ungrouped") + output = f"{var}" + + if var.endswith(".#"): + output += " (dynamic substitution allowed)" + + if alias_reverse.get(var): + alias_str = ", ".join(sorted(alias_reverse[var])) + output += f" [Alias: {alias_str}]" + + if bases.get(var): + base_str = ", ".join(bases[var]) + output += f" [Base: {base_str}]" + + if group != "ungrouped" and main_variations[var].get("default", False): + output += " [Default]" + + grouped_output[group].append(output) + + # Console output structure + print("\nVariations:\n") + for group in sorted(grouped_output): + print(f"\t{group.capitalize()} Variations:") + for line in grouped_output[group]: + print(f"\t - {line}") + print("") # Blank line between groups + + +def print_run_commands(metadata): + tags = ','.join(metadata.get('tags', [])) + input_mapping = metadata.get('input_mapping', {}) + + # Build the command using the extracted tags + command = f"mlcr {tags}" + print("\nRun Commands:\n") + print(f" $ {command}") + + print("") + print(f""" + * Inputs can be appended to the run command directly or as their --env.key mapping. + * --env.key is useful to modify the input of a dependent script for which direct input may not be there in the main script. + * --adr. can be used to modify the dependency(ies) with the name dep_name. + For example, --adr.compiler.tags=gcc adds the tags 'gcc' to any dependency under the name compiler. + """) + + +def infer_type(field): + if "dtype" in field: + return field["dtype"] + elif "default" in field: + return type(field["default"]).__name__ + else: + return "str" diff --git a/automation/script/module.py b/automation/script/module.py index 6c15f76c5..5adf05d67 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -529,12 +529,6 @@ def _run(self, i): if i.get(key): ii[key] = i[key] - # if cm run script without tags/artifact and with --help - if len(ii.get('parsed_artifact', [])) == 0 and ii.get( - 'tags', '') == '' and i.get('help', False): - return utils.call_internal_module( - self, __file__, 'module_help', 'print_help', {'meta': {}, 'path': ''}) - r = self.search(ii) if r['return'] > 0: return r @@ -791,8 +785,7 @@ def _run(self, i): # Check if has --help if i.get('help', False): - return utils.call_internal_module(self, __file__, 'module_help', 'print_help', { - 'meta': meta, 'path': path}) + return self.help(i) run_state['script_id'] = meta['alias'] + "," + meta['uid'] run_state['script_tags'] = script_tags @@ -4531,6 +4524,36 @@ def doc(self, i): ############################################################ + def help(self, i): + """ + Document MLC script. + + Args: + (MLC input dict): + + (out) (str): if 'con', output to console + + parsed_artifact (list): prepared in MLC CLI or MLC access function + [ (artifact alias, artifact UID) ] or + [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] + + (repos) (str): list of repositories to search for automations + + (output_dir) (str): output directory (../docs by default) + + Returns: + (MLC return dict): + + * return (int): return code == 0 if no error and >0 if error + * (error) (str): error string if return>0 + + """ + + from script.help import display_help + return display_help(self, i) + + ############################################################ + def lint(self, i): from script.lint import lint_meta return lint_meta(self, i) From 6c60c1fce6a96d802b8d75b49b12ec5ab9d3abce Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 27 Jul 2025 00:39:10 +0100 Subject: [PATCH 037/124] Improve script help (#543) --- automation/script/help.py | 113 +++++++++++++++++++++++++++++------- automation/script/module.py | 3 + 2 files changed, 94 insertions(+), 22 deletions(-) diff --git a/automation/script/help.py b/automation/script/help.py index c62fc790b..9c2212931 100644 --- a/automation/script/help.py +++ b/automation/script/help.py @@ -34,15 +34,24 @@ def display_help(self_module, input_params): if not scripts_list: return {'return': 1, 'error': 'No scripts were found'} + generic_inputs = self_module.input_flags_converted_to_env + # Step 4: Iterate over scripts and generate help output for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): metadata = script.meta - print_script_help(metadata) + script_path = script.path + print_script_help( + metadata, + script_path, + generic_inputs, + env, + self_module) return {'return': 0} -def print_script_help(metadata): +def print_script_help(metadata, script_path, generic_inputs, env, self_module): + print("") print(f"Script Name: {metadata.get('alias', metadata['uid'])}") print(f"Tags: {', '.join(metadata.get('tags', []))}") # Print the run commands @@ -55,11 +64,57 @@ def print_script_help(metadata): input_description = metadata.get('input_description', {}) default_env = metadata.get('default_env', {}) - reverse_map = defaultdict(list) - for k, v in input_mapping.items(): - reverse_map[v].append(k) + print_input_details(input_mapping, input_description, default_env) + + print("") + print("Generic Inputs for all Scripts:") + print("") + print_input_descriptions(generic_inputs) + + variations = metadata.get('variations', {}) + + if variations: + print_variations_help(variations) + else: + print(" - No variations.") + + print("\n" + "=" * 60 + "\n") # Separator for clarity + + print( + f"""Script meta file path: {os.path.join(script_path, "meta.yaml")}""") + customize_path = os.path.join(script_path, "customize.py") + if os.path.exists(customize_path): + print(f"""Script customize file path: {customize_path}""") + else: + print(f"""Script customize file can be created at: {customize_path}""") + + run_script_name = self_module._get_script_name(env, script_path) + run_script_path = os.path.join(script_path, run_script_name) + if os.path.exists(run_script_path): + print(f"""Script run file path: {run_script_path}""") + + print("\n" + "=" * 60 + "\n") # Separator for clarity + + +def print_input_details(input_mapping, input_description, default_env): + + for i in input_mapping: + if i not in input_description or input_description[i].get( + 'env', '') != input_mapping[i]: + if i not in input_description: + input_description[i] = {} + input_description[i]['env_key'] = input_mapping[i] + + keys_to_delete = [ + key for key in input_description if key not in input_mapping and "." not in key] + for key in keys_to_delete: + del input_description[key] if input_description: + reverse_map = defaultdict(list) + for k, v in input_mapping.items(): + reverse_map[v].append(k) + for i in input_description: if i in input_mapping and input_mapping[i] in default_env: input_description[i]['default'] = default_env[input_mapping[i]] @@ -72,24 +127,36 @@ def print_script_help(metadata): if alias in input_description: input_description[alias] = {} input_description[alias]['alias'] = canonical - input_description[alias]['desc'] = f"""Alias for {canonical}""" + input_description[alias]['desc'] = f"""Alias for --{canonical}""" + input_description[alias]['env_key'] = mapped_env - for key, value in input_mapping.items(): - desc = input_description.get(key, {}).get( - 'desc', 'No description available') - default = input_description.get(key, {}).get('default', 'None') - # Use .ljust(15) to ensure the key occupies 15 characters minimum - print( - f" --{key.ljust(26)} : maps to --env.{value}\n{' '.ljust(35)}{desc}\n{' '.ljust(35)}Default: {default}\n") + print_input_descriptions(input_description) - variations = metadata.get('variations', {}) + return - if variations: - print_variations_help(variations) - else: - print(" - No variations.") - print("\n" + "=" * 40 + "\n") # Separator for clarity +def print_input_descriptions(input_descriptions): + + if not input_descriptions: + print("\tNo inputs") + + for key in input_descriptions: + field = input_descriptions[key] + env_key = field.get('env_key', f"""MLC_TMP_{key.upper()}""") + desc = field.get('desc') + default = field.get('default', 'None') + choices = field.get("choices", "") + dtype = infer_type(field) + # Use .ljust(15) to ensure the key occupies 15 characters minimum + print(f"\t--{key.ljust(26)}: maps to --env.{env_key}") + if desc: + print(f"\t{' '.ljust(30)}Desc: {desc}") + print(f"\t{' '.ljust(30)}Default: {default}") + if choices: + print(f"\t{' '.ljust(30)}Choices: {choices}") + if dtype: + print(f"\t{' '.ljust(30)}Type: {dtype}") + print("") def print_variations_help(variations): @@ -139,7 +206,7 @@ def print_variations_help(variations): if bases.get(var): base_str = ", ".join(bases[var]) - output += f" [Base: {base_str}]" + output += f" [base: {base_str}]" if group != "ungrouped" and main_variations[var].get("default", False): output += " [Default]" @@ -166,10 +233,12 @@ def print_run_commands(metadata): print("") print(f""" - * Inputs can be appended to the run command directly or as their --env.key mapping. + * Any input can be appended to the run command directly or using its --env.key mapping. * --env.key is useful to modify the input of a dependent script for which direct input may not be there in the main script. + * Any variation can be selected by adding it to the tags using the _ prefix. + For example, mlcr get,generic-python-lib,_panda turns on the panda variation for the get-generic-python-lib script. * --adr. can be used to modify the dependency(ies) with the name dep_name. - For example, --adr.compiler.tags=gcc adds the tags 'gcc' to any dependency under the name compiler. + For example, --adr.compiler.tags=gcc adds the tag 'gcc' to any dependency with the name compiler. """) diff --git a/automation/script/module.py b/automation/script/module.py index 5adf05d67..bad6a68e9 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4630,6 +4630,9 @@ def clean_some_tmp_files(self, i): return {'return': 0} + def _get_script_name(self, env, path, filename="run"): + return get_script_name(env, path, filename) + def get_version_tag_from_version(version, cached_tags): tags_to_add = [] From 77472bc570b31e2ed10432a01a6a44c364b1b99b Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sun, 27 Jul 2025 05:09:33 +0530 Subject: [PATCH 038/124] Add logging in push results to GitHub (#541) --- script/push-mlperf-inference-results-to-github/customize.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/push-mlperf-inference-results-to-github/customize.py b/script/push-mlperf-inference-results-to-github/customize.py index b11bbd63a..6b123bb64 100644 --- a/script/push-mlperf-inference-results-to-github/customize.py +++ b/script/push-mlperf-inference-results-to-github/customize.py @@ -53,4 +53,9 @@ def postprocess(i): env['MLC_MLPERF_RESULTS_SYNC_DIR'] = env['MLC_GIT_REPO_CHECKOUT_PATH'] + logger = automation.logger + + logger.info( + f"\n\nPath to the locally synced submission directory: {env['MLC_MLPERF_RESULTS_SYNC_DIR']}\n\n") + return {'return': 0} From faca0c82641b12a3a4f8da4f2bf12c9c715e5817 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 27 Jul 2025 16:55:59 +0530 Subject: [PATCH 039/124] Fix for ignoring incorrect tmp cache entries --- automation/script/module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/automation/script/module.py b/automation/script/module.py index bad6a68e9..a2ab27753 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -1384,6 +1384,7 @@ def _run(self, i): 'script_alias': meta['alias'], 'extra_tags': ",".join(extra_cache_tags), 'tags': ','.join(tmp_tags), + 'exact_tags_match': True, 'meta': cached_meta, 'force': True} From e19692dd4c216aace3bbc8f72c5e350d302a66f0 Mon Sep 17 00:00:00 2001 From: kamieyy Date: Mon, 28 Jul 2025 23:04:24 +0700 Subject: [PATCH 040/124] Added support for Whisper external model download --- script/get-ml-model-whisper/run.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 script/get-ml-model-whisper/run.sh diff --git a/script/get-ml-model-whisper/run.sh b/script/get-ml-model-whisper/run.sh new file mode 100644 index 000000000..f0e290b83 --- /dev/null +++ b/script/get-ml-model-whisper/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# Default checkpoint path +CHECKPOINT_PATH=${MLC_ML_MODEL_WHISPER_PATH:-whisper-large-v3} + +git lfs install + +if [ ! -d "$CHECKPOINT_PATH" ]; then + git clone https://huggingface.co/openai/whisper-large-v3 "$CHECKPOINT_PATH" +fi + +cd "${CHECKPOINT_PATH}" +git checkout 06f233fe06e710322aca913c1bc4249a0d71fce1 From f40ee044e36c418a9c2f45fc3a2efe41e30703a5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 28 Jul 2025 23:34:43 +0530 Subject: [PATCH 041/124] Upgrade the docker image for mlperf inference reference implementation --- script/app-mlperf-inference/meta.yaml | 2 +- script/build-dockerfile/customize.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 16be0570f..551046175 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -1740,7 +1740,7 @@ variations: cuda,reference: docker: - base_image: nvcr.io/nvidia/pytorch:24.03-py3 + base_image: nvcr.io/nvidia/pytorch:24.08-py3 cuda: docker: diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 5b34a3f6c..ab0b38693 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -227,7 +227,7 @@ def preprocess(i): if env.get('MLC_DOCKER_EXTRA_SYS_DEPS', '') != '': f.write('RUN ' + env['MLC_DOCKER_EXTRA_SYS_DEPS'] + EOL) - if env['MLC_DOCKER_OS'] == "ubuntu": + if env['MLC_DOCKER_OS'] == "ubuntu" and False: #Turning this off as we are using venv if int(str(env['MLC_DOCKER_OS_VERSION']).split('.')[0]) >= 23: if "--break-system-packages" not in env.get( 'MLC_DOCKER_PIP_INSTALL_EXTRA_FLAGS', ''): From f9eca2f77768dc70b0d99381c96554793a395a40 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 28 Jul 2025 18:05:00 +0000 Subject: [PATCH 042/124] [Automated Commit] Format Codebase [skip ci] --- script/build-dockerfile/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index ab0b38693..987f7ba6f 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -227,7 +227,7 @@ def preprocess(i): if env.get('MLC_DOCKER_EXTRA_SYS_DEPS', '') != '': f.write('RUN ' + env['MLC_DOCKER_EXTRA_SYS_DEPS'] + EOL) - if env['MLC_DOCKER_OS'] == "ubuntu" and False: #Turning this off as we are using venv + if env['MLC_DOCKER_OS'] == "ubuntu" and False: # Turning this off as we are using venv if int(str(env['MLC_DOCKER_OS_VERSION']).split('.')[0]) >= 23: if "--break-system-packages" not in env.get( 'MLC_DOCKER_PIP_INSTALL_EXTRA_FLAGS', ''): From 9fe98883f0f93ad704301f9524fb6aa1dd607144 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 01:19:13 +0530 Subject: [PATCH 043/124] Remove version fix for pycuda, support cuda 12.8, 12.9 --- script/app-mlperf-inference/meta.yaml | 4 ++-- script/install-cuda-prebuilt/meta.yaml | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 551046175..bc6461a5a 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -2116,7 +2116,7 @@ variations: nvidia-mitten-git-src: tags: _sha.e5574192b3ae76910b6824a9275d14a4158d8afa pycuda: - version: "2024.1" + version_min: "2024.1" default_env: MLC_SKIP_SYS_UTILS: 'yes' MLC_REGENERATE_MEASURE_FILES: 'yes' @@ -2140,7 +2140,7 @@ variations: nvidia-mitten-git-src: tags: _sha.e5574192b3ae76910b6824a9275d14a4158d8afa pycuda: - version: "2024.1" + version_min: "2024.1" default_env: MLC_SKIP_SYS_UTILS: 'yes' MLC_REGENERATE_MEASURE_FILES: 'yes' diff --git a/script/install-cuda-prebuilt/meta.yaml b/script/install-cuda-prebuilt/meta.yaml index a5f06022e..2d0f1ce1e 100644 --- a/script/install-cuda-prebuilt/meta.yaml +++ b/script/install-cuda-prebuilt/meta.yaml @@ -8,6 +8,7 @@ default_env: default_version: 11.8.0 deps: - tags: detect,os +- tags: detect,sudo docker: run: true input_mapping: @@ -15,6 +16,7 @@ input_mapping: local_run_file_path: CUDA_RUN_FILE_LOCAL_PATH override-driver-check: MLC_CUDA_DRIVER_INSTALL_OVERRIDE skip_sudo: CUDA_SKIP_SUDO +sudo_install: True new_env_keys: - MLC_CUDA_* - MLC_NVCC_* @@ -89,3 +91,9 @@ versions: 12.6.1: env: MLC_CUDA_LINUX_FILENAME: cuda_12.6.1_560.35.03_linux.run + 12.8.1: + env: + MLC_CUDA_LINUX_FILENAME: cuda_12.8.1_570.124.06_linux.run + 12.9.1: + env: + MLC_CUDA_LINUX_FILENAME: cuda_12.9.1_575.57.08_linux.run From 3a17bb98e7cbce5b0abc957c2869e99f53a59f99 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 04:03:52 +0530 Subject: [PATCH 044/124] Fixes for whisper, dont print python env at end --- script/app-mlperf-inference-mlcommons-python/meta.yaml | 3 +-- script/get-python3/meta.yaml | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 197d8ec94..551fe922c 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -1510,7 +1510,6 @@ variations: - tags: get,generic-python-lib,_package.setuptools-scm - tags: get,generic-python-lib,_package.librosa - tags: get,generic-python-lib,_package.transformers - version_max: "4.52.4" - tags: get,generic-python-lib,_package.openai-whisper # - tags: get,generic-python-lib,_package.evaluate # - tags: get,generic-python-lib,_package.absl-py @@ -1521,7 +1520,7 @@ variations: - "yes" names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_r2_downloader + - tags: get,dataset,whisper,_preprocessed names: - whisper-dataset skip_if_env: diff --git a/script/get-python3/meta.yaml b/script/get-python3/meta.yaml index b187346d2..434a00f0e 100644 --- a/script/get-python3/meta.yaml +++ b/script/get-python3/meta.yaml @@ -21,9 +21,6 @@ prehook_deps: inherit_variation_tags: true reuse_version: true tags: install,python,src -print_env_at_the_end: - MLC_PYTHON_BIN_WITH_PATH: Path to Python - MLC_PYTHON_VERSION: Python version tags: - get - python From fbd86f0ef63fbda8e4c5f70d8502eb498b11c271 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 04:47:11 +0530 Subject: [PATCH 045/124] adr -> ad in get-ml-model-whisper --- script/get-ml-model-whisper/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index bd7f4eaef..1003c365a 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -52,7 +52,7 @@ variations: _url.: - MLC_DOWNLOAD_URL rclone: - add_deps_recursive: + add_deps: dae: tags: _rclone env: @@ -72,9 +72,9 @@ variations: default: true group: download-tool r2_downloader: - add_deps_recursive: + add_deps: dae: tags: _r2_downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' - group: download-tool \ No newline at end of file + group: download-tool From b840cf26d78e688ded8b657aa408b06851c50a24 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 04:49:15 +0530 Subject: [PATCH 046/124] r2_downloader -> r2-downloader --- .../meta.yaml | 10 +++++----- script/app-mlperf-inference/meta.yaml | 12 ++++++------ script/download-and-extract/meta.yaml | 4 ++-- script/download-file/meta.yaml | 4 ++-- script/get-dataset-cnndm/meta.yaml | 18 +++++++++--------- script/get-dataset-whisper/meta.yaml | 10 +++++----- script/get-ml-model-llama3/meta.yaml | 12 ++++++------ script/get-ml-model-whisper/meta.yaml | 8 ++++---- .../meta.yaml | 12 ++++++------ 9 files changed, 45 insertions(+), 45 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 551fe922c..7980fa9f1 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -526,7 +526,7 @@ deps: - "yes" ## LLAMA3_1-8B - - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader + - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader names: - llama3-8b-model enable_if_env: @@ -1476,7 +1476,7 @@ variations: MLC_MODEL: llama3_1-8b deps: ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter skip_if_env: @@ -1490,7 +1490,7 @@ variations: MLC_MODEL: llama3_1-8b-edge deps: ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge skip_if_env: @@ -1514,7 +1514,7 @@ variations: # - tags: get,generic-python-lib,_package.evaluate # - tags: get,generic-python-lib,_package.absl-py # - tags: get,generic-python-lib,_package.rouge-score - - tags: get,ml-model,whisper,_r2_downloader,_mlc + - tags: get,ml-model,whisper,_r2-downloader,_mlc skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" @@ -1532,7 +1532,7 @@ variations: env: MLC_MODEL: deepseek-r1 deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2_downloader + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2-downloader names: - deepseek-r1-preprocessed-dataset skip_if_env: diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index bc6461a5a..cfcb5ef1a 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -969,7 +969,7 @@ variations: - "${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}" - "${{ MLC_DATASET_CNNDM_EVAL_PATH }}:${{ MLC_DATASET_CNNDM_EVAL_PATH }}" deps: - - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader + - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' @@ -1003,7 +1003,7 @@ variations: docker: deps: ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter enable_if_env: @@ -1038,7 +1038,7 @@ variations: docker: deps: ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge enable_if_env: @@ -1072,13 +1072,13 @@ variations: tags: run,accuracy,mlperf,_librispeech_whisper,_int64 docker: deps: - - tags: get,ml-model,whisper,_r2_downloader,_mlc + - tags: get,ml-model,whisper,_r2-downloader,_mlc enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_r2_downloader + - tags: get,dataset,whisper,_preprocessed,_mlc,_r2-downloader names: - whisper-dataset enable_if_env: @@ -1150,7 +1150,7 @@ variations: tags: run,accuracy,mlperf,_dataset_deepseek-r1 docker: deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2_downloader + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2-downloader enable_if_env: MLC_USE_DATASET_FROM_HOST: - 'yes' diff --git a/script/download-and-extract/meta.yaml b/script/download-and-extract/meta.yaml index 02aaf1175..8a3b17d07 100644 --- a/script/download-and-extract/meta.yaml +++ b/script/download-and-extract/meta.yaml @@ -125,9 +125,9 @@ variations: download-script: tags: _wget group: download-tool - r2_downloader: + r2-downloader: add_deps: download-script: - tags: _r2_downloader + tags: _r2-downloader group: download-tool versions: {} diff --git a/script/download-file/meta.yaml b/script/download-file/meta.yaml index e195b078f..aaf9bf1dd 100644 --- a/script/download-file/meta.yaml +++ b/script/download-file/meta.yaml @@ -77,9 +77,9 @@ variations: env: MLC_DOWNLOAD_TOOL: wget group: download-tool - r2_downloader: + r2-downloader: env: - MLC_DOWNLOAD_TOOL: r2_downloader + MLC_DOWNLOAD_TOOL: r2-downloader group: download-tool versions: {} diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index 4d38491f8..1bdf3db37 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -72,11 +72,11 @@ variations: add_deps_recursive: dae: tags: _rclone - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader dry-run: group: run-mode env: @@ -84,7 +84,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -135,7 +135,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH - validation,edge,llama3,mlc,r2_downloader: + validation,edge,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,edge @@ -153,7 +153,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH - validation,datacenter,llama3,mlc,r2_downloader: + validation,datacenter,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,datacenter @@ -171,7 +171,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH - calibation,llama3,mlc,r2_downloader: + calibation,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,calib @@ -185,7 +185,7 @@ tests: - variations_list: # - validation,edge,rclone,llama3,mlc,dry-run # - validation,datacenter,rclone,llama3,mlc,dry-run - - validation,edge,r2_downloader,llama3,mlc,dry-run - - validation,datacenter,r2_downloader,llama3,mlc,dry-run - - calibration,r2_downloader,llama3,mlc,dry-run + - validation,edge,r2-downloader,llama3,mlc,dry-run + - validation,datacenter,r2-downloader,llama3,mlc,dry-run + - calibration,r2-downloader,llama3,mlc,dry-run # - calibration,rclone,llama3,mlc,dry-run diff --git a/script/get-dataset-whisper/meta.yaml b/script/get-dataset-whisper/meta.yaml index 12fddf9e4..a436e6ef5 100644 --- a/script/get-dataset-whisper/meta.yaml +++ b/script/get-dataset-whisper/meta.yaml @@ -14,7 +14,7 @@ tests: run_inputs: - variations_list: - rclone,preprocessed,mlc,dry-run - - r2_downloader,preprocessed,mlc,dry-run + - r2-downloader,preprocessed,mlc,dry-run variations: preprocessed: group: dataset-type @@ -45,7 +45,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -59,10 +59,10 @@ variations: tags: _rclone default: true group: download-tool - r2_downloader: + r2-downloader: add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader group: download-tool rclone,preprocessed: env: @@ -95,6 +95,6 @@ variations: update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL - r2_downloader,preprocessed: + r2-downloader,preprocessed: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/whisper-dataset.uri diff --git a/script/get-ml-model-llama3/meta.yaml b/script/get-ml-model-llama3/meta.yaml index fd79fdc33..8b26d7fff 100644 --- a/script/get-ml-model-llama3/meta.yaml +++ b/script/get-ml-model-llama3/meta.yaml @@ -38,8 +38,8 @@ tests: run_inputs: - variations_list: - rclone,405b,mlc,dry-run - - r2_downloader,405b,mlc,dry-run - - r2_downloader,8b,mlc,dry-run + - r2-downloader,405b,mlc,dry-run + - r2-downloader,8b,mlc,dry-run variations: fp16: default: true @@ -86,7 +86,7 @@ variations: adr: dae: extra_cache_tags: llama3,dataset,rclone - mlc,r2_downloader: + mlc,r2-downloader: env: MLC_DOWNLOAD_URL: https://llama3-1.mlcommons-storage.org/metadata/<<>>.uri adr: @@ -108,11 +108,11 @@ variations: MLC_TMP_REQUIRE_DOWNLOAD: - yes default: true - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader dry-run: group: run-mode env: @@ -120,7 +120,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x hf: diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index 1003c365a..ece8ec1fc 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -16,7 +16,7 @@ tests: run_inputs: - variations_list: - rclone,mlc,dry-run - - r2_downloader,mlc,dry-run + - r2-downloader,mlc,dry-run uid: 3bea2356e97f47b1 variations: dry-run: @@ -26,7 +26,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -71,10 +71,10 @@ variations: tags: get,rclone-config,_mlc-inference default: true group: download-tool - r2_downloader: + r2-downloader: add_deps: dae: - tags: _r2_downloader + tags: _r2-downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' group: download-tool diff --git a/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml b/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml index 8c0da0989..0cc5f359c 100644 --- a/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml +++ b/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml @@ -28,18 +28,18 @@ variations: validation,rclone: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/deepseek_r1/mlperf_deepseek_r1_dataset_4388_fp8_eval.pkl - validation,r2_downloader: + validation,r2-downloader: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-dataset-4388-fp8-eval.uri calibration: group: dataset-type env: MLC_PREPROCESSED_DATASET_TYPE: calibration - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader rclone: group: download-tool add_deps_recursive: @@ -66,7 +66,7 @@ variations: calibration,rclone: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/deepseek_r1/mlperf_deepseek_r1_calibration_dataset_500_fp8_eval.pkl - calibration,r2_downloader: + calibration,r2-downloader: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-calibration-dataset-500-fp8-eval.uri mlc: @@ -94,8 +94,8 @@ variations: tests: run_inputs: - variations_list: - - calibration,r2_downloader,mlc,dry-run - - validation,r2_downloader,mlc,dry-run + - calibration,r2-downloader,mlc,dry-run + - validation,r2-downloader,mlc,dry-run # - validation,rclone,mlc,dry-run # - calibration,rclone,mlc,dry-run From 39fa33dbb7e1b5dc731f28fcb447960eaed773e6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 04:56:12 +0530 Subject: [PATCH 047/124] adr fix in get-ml-model-whisper --- script/get-ml-model-whisper/meta.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index ece8ec1fc..de84c768b 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -46,14 +46,14 @@ variations: force_env_keys: - MLC_OUTDIRNAME names: - - dae + - whisper-model-dae tags: download-and-extract update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL rclone: - add_deps: - dae: + adr: + whisper-model-dae: tags: _rclone env: MLC_DOWNLOAD_URL: 'mlc-inference:mlcommons-inference-wg-public/Whisper/model/' @@ -72,8 +72,8 @@ variations: default: true group: download-tool r2-downloader: - add_deps: - dae: + adr: + whisper-model-dae: tags: _r2-downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' From e8c71030b374d904e19dd197a5810639f4855e43 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 00:29:31 +0100 Subject: [PATCH 048/124] Fixes for whisper, dont print python env at end (#551) --- .../meta.yaml | 13 ++++++------ script/app-mlperf-inference/meta.yaml | 12 +++++------ script/download-and-extract/meta.yaml | 4 ++-- script/download-file/meta.yaml | 4 ++-- script/get-dataset-cnndm/meta.yaml | 18 ++++++++--------- script/get-dataset-whisper/meta.yaml | 10 +++++----- script/get-ml-model-llama3/meta.yaml | 12 +++++------ script/get-ml-model-whisper/meta.yaml | 20 +++++++++---------- .../meta.yaml | 12 +++++------ script/get-python3/meta.yaml | 3 --- 10 files changed, 52 insertions(+), 56 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 197d8ec94..7980fa9f1 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -526,7 +526,7 @@ deps: - "yes" ## LLAMA3_1-8B - - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader + - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader names: - llama3-8b-model enable_if_env: @@ -1476,7 +1476,7 @@ variations: MLC_MODEL: llama3_1-8b deps: ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter skip_if_env: @@ -1490,7 +1490,7 @@ variations: MLC_MODEL: llama3_1-8b-edge deps: ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge skip_if_env: @@ -1510,18 +1510,17 @@ variations: - tags: get,generic-python-lib,_package.setuptools-scm - tags: get,generic-python-lib,_package.librosa - tags: get,generic-python-lib,_package.transformers - version_max: "4.52.4" - tags: get,generic-python-lib,_package.openai-whisper # - tags: get,generic-python-lib,_package.evaluate # - tags: get,generic-python-lib,_package.absl-py # - tags: get,generic-python-lib,_package.rouge-score - - tags: get,ml-model,whisper,_r2_downloader,_mlc + - tags: get,ml-model,whisper,_r2-downloader,_mlc skip_if_env: MLC_RUN_STATE_DOCKER: - "yes" names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_r2_downloader + - tags: get,dataset,whisper,_preprocessed names: - whisper-dataset skip_if_env: @@ -1533,7 +1532,7 @@ variations: env: MLC_MODEL: deepseek-r1 deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2_downloader + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2-downloader names: - deepseek-r1-preprocessed-dataset skip_if_env: diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index bc6461a5a..cfcb5ef1a 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -969,7 +969,7 @@ variations: - "${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}:${{ MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH }}" - "${{ MLC_DATASET_CNNDM_EVAL_PATH }}:${{ MLC_DATASET_CNNDM_EVAL_PATH }}" deps: - - tags: get,ml-model,llama3,_mlc,_8b,_r2_downloader + - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' @@ -1003,7 +1003,7 @@ variations: docker: deps: ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter enable_if_env: @@ -1038,7 +1038,7 @@ variations: docker: deps: ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2_downloader + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge enable_if_env: @@ -1072,13 +1072,13 @@ variations: tags: run,accuracy,mlperf,_librispeech_whisper,_int64 docker: deps: - - tags: get,ml-model,whisper,_r2_downloader,_mlc + - tags: get,ml-model,whisper,_r2-downloader,_mlc enable_if_env: MLC_USE_ML_MODEL_FROM_HOST: - 'yes' names: - whisper-model - - tags: get,dataset,whisper,_preprocessed,_mlc,_r2_downloader + - tags: get,dataset,whisper,_preprocessed,_mlc,_r2-downloader names: - whisper-dataset enable_if_env: @@ -1150,7 +1150,7 @@ variations: tags: run,accuracy,mlperf,_dataset_deepseek-r1 docker: deps: - - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2_downloader + - tags: get,preprocessed,dataset,deepseek-r1,_validation,_mlc,_r2-downloader enable_if_env: MLC_USE_DATASET_FROM_HOST: - 'yes' diff --git a/script/download-and-extract/meta.yaml b/script/download-and-extract/meta.yaml index 02aaf1175..8a3b17d07 100644 --- a/script/download-and-extract/meta.yaml +++ b/script/download-and-extract/meta.yaml @@ -125,9 +125,9 @@ variations: download-script: tags: _wget group: download-tool - r2_downloader: + r2-downloader: add_deps: download-script: - tags: _r2_downloader + tags: _r2-downloader group: download-tool versions: {} diff --git a/script/download-file/meta.yaml b/script/download-file/meta.yaml index e195b078f..aaf9bf1dd 100644 --- a/script/download-file/meta.yaml +++ b/script/download-file/meta.yaml @@ -77,9 +77,9 @@ variations: env: MLC_DOWNLOAD_TOOL: wget group: download-tool - r2_downloader: + r2-downloader: env: - MLC_DOWNLOAD_TOOL: r2_downloader + MLC_DOWNLOAD_TOOL: r2-downloader group: download-tool versions: {} diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index 4d38491f8..1bdf3db37 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -72,11 +72,11 @@ variations: add_deps_recursive: dae: tags: _rclone - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader dry-run: group: run-mode env: @@ -84,7 +84,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -135,7 +135,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH - validation,edge,llama3,mlc,r2_downloader: + validation,edge,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,edge @@ -153,7 +153,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH - validation,datacenter,llama3,mlc,r2_downloader: + validation,datacenter,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,datacenter @@ -171,7 +171,7 @@ variations: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH - calibation,llama3,mlc,r2_downloader: + calibation,llama3,mlc,r2-downloader: adr: dae: extra_cache_tags: cnndm,dataset,llama3,calib @@ -185,7 +185,7 @@ tests: - variations_list: # - validation,edge,rclone,llama3,mlc,dry-run # - validation,datacenter,rclone,llama3,mlc,dry-run - - validation,edge,r2_downloader,llama3,mlc,dry-run - - validation,datacenter,r2_downloader,llama3,mlc,dry-run - - calibration,r2_downloader,llama3,mlc,dry-run + - validation,edge,r2-downloader,llama3,mlc,dry-run + - validation,datacenter,r2-downloader,llama3,mlc,dry-run + - calibration,r2-downloader,llama3,mlc,dry-run # - calibration,rclone,llama3,mlc,dry-run diff --git a/script/get-dataset-whisper/meta.yaml b/script/get-dataset-whisper/meta.yaml index 12fddf9e4..a436e6ef5 100644 --- a/script/get-dataset-whisper/meta.yaml +++ b/script/get-dataset-whisper/meta.yaml @@ -14,7 +14,7 @@ tests: run_inputs: - variations_list: - rclone,preprocessed,mlc,dry-run - - r2_downloader,preprocessed,mlc,dry-run + - r2-downloader,preprocessed,mlc,dry-run variations: preprocessed: group: dataset-type @@ -45,7 +45,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -59,10 +59,10 @@ variations: tags: _rclone default: true group: download-tool - r2_downloader: + r2-downloader: add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader group: download-tool rclone,preprocessed: env: @@ -95,6 +95,6 @@ variations: update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL - r2_downloader,preprocessed: + r2-downloader,preprocessed: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/whisper-dataset.uri diff --git a/script/get-ml-model-llama3/meta.yaml b/script/get-ml-model-llama3/meta.yaml index fd79fdc33..8b26d7fff 100644 --- a/script/get-ml-model-llama3/meta.yaml +++ b/script/get-ml-model-llama3/meta.yaml @@ -38,8 +38,8 @@ tests: run_inputs: - variations_list: - rclone,405b,mlc,dry-run - - r2_downloader,405b,mlc,dry-run - - r2_downloader,8b,mlc,dry-run + - r2-downloader,405b,mlc,dry-run + - r2-downloader,8b,mlc,dry-run variations: fp16: default: true @@ -86,7 +86,7 @@ variations: adr: dae: extra_cache_tags: llama3,dataset,rclone - mlc,r2_downloader: + mlc,r2-downloader: env: MLC_DOWNLOAD_URL: https://llama3-1.mlcommons-storage.org/metadata/<<>>.uri adr: @@ -108,11 +108,11 @@ variations: MLC_TMP_REQUIRE_DOWNLOAD: - yes default: true - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader dry-run: group: run-mode env: @@ -120,7 +120,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x hf: diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index bd7f4eaef..de84c768b 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -16,7 +16,7 @@ tests: run_inputs: - variations_list: - rclone,mlc,dry-run - - r2_downloader,mlc,dry-run + - r2-downloader,mlc,dry-run uid: 3bea2356e97f47b1 variations: dry-run: @@ -26,7 +26,7 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run - dry-run,r2_downloader: + dry-run,r2-downloader: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: @@ -46,14 +46,14 @@ variations: force_env_keys: - MLC_OUTDIRNAME names: - - dae + - whisper-model-dae tags: download-and-extract update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL rclone: - add_deps_recursive: - dae: + adr: + whisper-model-dae: tags: _rclone env: MLC_DOWNLOAD_URL: 'mlc-inference:mlcommons-inference-wg-public/Whisper/model/' @@ -71,10 +71,10 @@ variations: tags: get,rclone-config,_mlc-inference default: true group: download-tool - r2_downloader: - add_deps_recursive: - dae: - tags: _r2_downloader + r2-downloader: + adr: + whisper-model-dae: + tags: _r2-downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' - group: download-tool \ No newline at end of file + group: download-tool diff --git a/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml b/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml index 8c0da0989..0cc5f359c 100644 --- a/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml +++ b/script/get-preprocessed-dataset-mlperf-deepseek-r1/meta.yaml @@ -28,18 +28,18 @@ variations: validation,rclone: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/deepseek_r1/mlperf_deepseek_r1_dataset_4388_fp8_eval.pkl - validation,r2_downloader: + validation,r2-downloader: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-dataset-4388-fp8-eval.uri calibration: group: dataset-type env: MLC_PREPROCESSED_DATASET_TYPE: calibration - r2_downloader: + r2-downloader: group: download-tool add_deps_recursive: dae: - tags: _r2_downloader + tags: _r2-downloader rclone: group: download-tool add_deps_recursive: @@ -66,7 +66,7 @@ variations: calibration,rclone: env: MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/deepseek_r1/mlperf_deepseek_r1_calibration_dataset_500_fp8_eval.pkl - calibration,r2_downloader: + calibration,r2-downloader: env: MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/deepseek-r1-calibration-dataset-500-fp8-eval.uri mlc: @@ -94,8 +94,8 @@ variations: tests: run_inputs: - variations_list: - - calibration,r2_downloader,mlc,dry-run - - validation,r2_downloader,mlc,dry-run + - calibration,r2-downloader,mlc,dry-run + - validation,r2-downloader,mlc,dry-run # - validation,rclone,mlc,dry-run # - calibration,rclone,mlc,dry-run diff --git a/script/get-python3/meta.yaml b/script/get-python3/meta.yaml index b187346d2..434a00f0e 100644 --- a/script/get-python3/meta.yaml +++ b/script/get-python3/meta.yaml @@ -21,9 +21,6 @@ prehook_deps: inherit_variation_tags: true reuse_version: true tags: install,python,src -print_env_at_the_end: - MLC_PYTHON_BIN_WITH_PATH: Path to Python - MLC_PYTHON_VERSION: Python version tags: - get - python From d7119a227fc621462eb04b2b33b337926a89dfbc Mon Sep 17 00:00:00 2001 From: kamieyy Date: Tue, 29 Jul 2025 10:11:37 +0700 Subject: [PATCH 049/124] Updated meta.yaml to align with repo structure and automation --- script/get-ml-model-whisper/meta.yaml | 116 ++++++++++++++------------ 1 file changed, 61 insertions(+), 55 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index bd7f4eaef..7ab434198 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -2,23 +2,78 @@ alias: get-ml-model-whisper automation_alias: script automation_uid: 5b4e0237da074764 cache: true +category: AI/ML models +env: + MLC_ML_MODEL_WEIGHT_TRANSFORMATIONS: 'no' + MLC_ML_MODEL_DATASET: whisper +input_mapping: + checkpoint: WHISPER_CHECKPOINT_PATH new_env_keys: -- MLC_ML_MODEL_WHISPER_PATH -- MLC_ML_MODEL_FILE_WITH_PATH +- MLC_ML_MODEL_* +- WHISPER_CHECKPOINT_PATH +- MLC_WHISPER_FINAL_SAFE_TENSORS_PATH print_env_at_the_end: - MLC_ML_MODEL_WHISPER_PATH: Whisper checkpoint path + WHISPER_CHECKPOINT_PATH: Whisper checkpoint path tags: - get-ml-model-whisper - get - ml-model - whisper +- speech-recognition tests: run_inputs: - variations_list: - - rclone,mlc,dry-run - - r2_downloader,mlc,dry-run + - huggingface,dry-run uid: 3bea2356e97f47b1 + +prehook_deps: +- enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + MLC_DOWNLOAD_SRC: + - 'huggingface' + env: {} + extra_cache_tags: whisper + force_env_keys: + - MLC_GIT_CHECKOUT_FOLDER + names: + - hf-zoo + tags: get,ml-model,huggingface,zoo,_clone-repo + force_env_keys: + - MLC_OUTDIRNAME + variations: + fp32: + default: true + env: + MLC_ML_MODEL_INPUT_DATA_TYPES: fp32 + MLC_ML_MODEL_PRECISION: fp32 + MLC_ML_MODEL_WEIGHT_DATA_TYPES: fp32 + group: precision + + huggingface: + group: download-source + env: + MLC_DOWNLOAD_SRC: huggingface + + large-v3: + env: + MLC_GIT_CHECKOUT_FOLDER: whisper-large-v3 + group: model-size + default: true + default_variations: + huggingface-stub: openai/whisper-large-v3 + + openai/whisper-large-v3: + base: + - large-v3 + adr: + hf-zoo: + tags: _model-stub.openai/whisper-large-v3 + env: + MLC_MODEL_ZOO_ENV_KEY: WHISPER + group: huggingface-stub + dry-run: env: MLC_DOWNLOAD_MODE: dry @@ -28,53 +83,4 @@ variations: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run dry-run,r2_downloader: env: - MLC_DOWNLOAD_EXTRA_OPTIONS: -x - mlc: - default: true - env: - MLC_DOWNLOAD_SRC: mlcommons - group: download-src - prehook_deps: - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - 'yes' - env: - MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_WHISPER_PATH - MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_WHISPER_PATH - extra_cache_tags: ml,model,whisper - force_cache: true - force_env_keys: - - MLC_OUTDIRNAME - names: - - dae - tags: download-and-extract - update_tags_from_env_with_prefix: - _url.: - - MLC_DOWNLOAD_URL - rclone: - add_deps_recursive: - dae: - tags: _rclone - env: - MLC_DOWNLOAD_URL: 'mlc-inference:mlcommons-inference-wg-public/Whisper/model/' - prehook_deps: - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - tags: get,rclone - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC - force_cache: true - tags: get,rclone-config,_mlc-inference - default: true - group: download-tool - r2_downloader: - add_deps_recursive: - dae: - tags: _r2_downloader - env: - MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' - group: download-tool \ No newline at end of file + MLC_DOWNLOAD_EXTRA_OPTIONS: -x \ No newline at end of file From 8d79c4bdbd3cd5dc35119afba8fcb2e6a446ea1e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Jul 2025 11:55:24 +0530 Subject: [PATCH 050/124] Changes to support R2 Downloader (#548) --- .../meta.yaml | 24 +++++++ script/get-dataset-waymo/meta.yaml | 34 +++++++--- script/get-ml-model-llama2/customize.py | 19 ++---- script/get-ml-model-llama2/meta.yaml | 65 +++++++++++++++++-- script/get-ml-model-llama2/run-rclone.sh | 4 -- script/get-ml-model-llama3/meta.yaml | 42 ++++++------ .../get-ml-model-pointpainting/customize.py | 40 +++++------- script/get-ml-model-pointpainting/meta.yaml | 51 +++++++++++++-- .../get-ml-model-pointpainting/run-rclone.sh | 4 -- 9 files changed, 200 insertions(+), 83 deletions(-) delete mode 100644 script/get-ml-model-llama2/run-rclone.sh delete mode 100644 script/get-ml-model-pointpainting/run-rclone.sh diff --git a/script/get-dataset-mlperf-inference-llama3/meta.yaml b/script/get-dataset-mlperf-inference-llama3/meta.yaml index 9e8762f35..24aef9e20 100644 --- a/script/get-dataset-mlperf-inference-llama3/meta.yaml +++ b/script/get-dataset-mlperf-inference-llama3/meta.yaml @@ -36,12 +36,28 @@ variations: MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl MLC_DATASET_TYPE: validation MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + validation,rclone: + env: + MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_dataset_8313_processed_fp16_eval.pkl + validation,r2-downloader: + env: + MLC_RCLONE_URL: "" + MLC_DATASET_FILE_NAME: "" calibration: group: dataset-type env: MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl MLC_DATASET_TYPE: calibration MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + calibration,rclone: + env: + MLC_RCLONE_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_405b/mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + MLC_DATASET_FILE_NAME: mlperf_llama3.1_405b_calibration_dataset_512_processed_fp16_eval.pkl + calibration,r2-downloader: + env: + MLC_RCLONE_URL: "" + MLC_DATASET_FILE_NAME: "" rclone: add_deps_recursive: dae: @@ -52,5 +68,13 @@ variations: MLC_DOWNLOAD_URL: <<>> MLC_RCLONE_CONFIG_NAME: mlc-inference group: download-tool + r2-downloader: + add_deps_recursive: + dae: + tags: _r2-downloader + env: + MLC_DOWNLOAD_FILENAME: checkpoint + MLC_RCLONE_CONFIG_NAME: mlc-inference + group: download-tool print_env_at_the_end: MLC_DATASET_LLAMA3_PATH: Path to the dataset diff --git a/script/get-dataset-waymo/meta.yaml b/script/get-dataset-waymo/meta.yaml index ef56e4a2e..e90315017 100644 --- a/script/get-dataset-waymo/meta.yaml +++ b/script/get-dataset-waymo/meta.yaml @@ -21,22 +21,12 @@ variations: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_waymo - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' env: MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_WAYMO_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_WAYMO_PATH - MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format extra_cache_tags: waymo,dataset force_cache: true names: @@ -54,6 +44,21 @@ variations: add_deps_recursive: dae: tags: _rclone + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_waymo + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader default: true dry-run: group: run-mode @@ -62,3 +67,12 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + rclone,mlc: + env: + MLC_DOWNLOAD_URL: mlc_waymo:waymo_preprocessed_dataset/kitti_format + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/dataset.uri diff --git a/script/get-ml-model-llama2/customize.py b/script/get-ml-model-llama2/customize.py index 494b7c9cf..491e76248 100644 --- a/script/get-ml-model-llama2/customize.py +++ b/script/get-ml-model-llama2/customize.py @@ -29,12 +29,6 @@ def preprocess(i): else: if path == '' or not os.path.exists(path): env['MLC_TMP_REQUIRE_DOWNLOAD'] = 'yes' - if env['MLC_DOWNLOAD_SRC'] == "mlcommons": - i['run_script_input']['script_name'] = 'run-rclone' - if env.get('MLC_OUTDIRNAME', '') != '': - env['LLAMA2_CHECKPOINT_PATH'] = env['MLC_OUTDIRNAME'] - else: - env['LLAMA2_CHECKPOINT_PATH'] = os.getcwd() return {'return': 0} @@ -42,11 +36,12 @@ def preprocess(i): def postprocess(i): env = i['env'] - if env.get('LLAMA2_CHECKPOINT_PATH', '') == '': - env['LLAMA2_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] - else: - env['MLC_ML_MODEL_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] - env['MLC_ML_MODEL_LLAMA2_FILE_WITH_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] - env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] + if env.get('MLC_DOWNLOAD_MODE', '') != "dry": + if env.get('LLAMA2_CHECKPOINT_PATH', '') == '': + env['LLAMA2_CHECKPOINT_PATH'] = env['MLC_ML_MODEL_PATH'] + else: + env['MLC_ML_MODEL_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] + env['MLC_ML_MODEL_LLAMA2_FILE_WITH_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] + env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_ML_MODEL_PATH'] return {'return': 0} diff --git a/script/get-ml-model-llama2/meta.yaml b/script/get-ml-model-llama2/meta.yaml index d0e14cf77..a2b0c3c1e 100644 --- a/script/get-ml-model-llama2/meta.yaml +++ b/script/get-ml-model-llama2/meta.yaml @@ -41,6 +41,12 @@ tags: - llama2-70b - text-summarization uid: 5db97be9f61244c6 +tests: + needs_pat: true + run_inputs: + - variations_list: + - r2-downloader,70b,mlc,dry-run + - r2-downloader,7b,mlc,dry-run variations: L40s: env: @@ -84,11 +90,11 @@ variations: MLC_ML_MODEL_PRECISION: int8 MLC_ML_MODEL_WEIGHT_DATA_TYPES: int8 group: precision - mlc: - group: download-source - default: true - env: - MLC_DOWNLOAD_SRC: mlcommons + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone prehook_deps: - tags: get,rclone enable_if_env: @@ -99,6 +105,55 @@ variations: enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - yes + r2-downloader: + group: download-tool + default: true + add_deps_recursive: + dae: + tags: _r2-downloader + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + mlc: + group: download-source + default: true + env: + MLC_DOWNLOAD_SRC: mlcommons + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: LLAMA2_CHECKPOINT_PATH + MLC_EXTRACT_FINAL_ENV_NAME: LLAMA2_CHECKPOINT_PATH + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + mlc,rclone,70b: + env: + MLC_DOWNLOAD_URL: mlc-llama2:Llama-2-70b-chat-hf + mlc,rclone,7b: + env: + MLC_DOWNLOAD_URL: mlc-llama2:Llama-2-7b-chat-hf + mlc,r2-downloader,70b: + env: + MLC_DOWNLOAD_URL: https://llama2.mlcommons-storage.org/metadata/llama-2-70b-chat-hf.uri + mlc,r2-downloader,7b: + env: + MLC_DOWNLOAD_URL: https://llama2.mlcommons-storage.org/metadata/llama-2-7b-chat-hf.uri hf: group: download-source env: diff --git a/script/get-ml-model-llama2/run-rclone.sh b/script/get-ml-model-llama2/run-rclone.sh deleted file mode 100644 index 1fc602a9f..000000000 --- a/script/get-ml-model-llama2/run-rclone.sh +++ /dev/null @@ -1,4 +0,0 @@ -cmd="rclone sync mlc-llama2:${MLC_GIT_CHECKOUT_FOLDER} ${LLAMA2_CHECKPOINT_PATH}/${MLC_GIT_CHECKOUT_FOLDER} -P" -echo $cmd -eval $cmd -test $? -eq 0 || exit $? diff --git a/script/get-ml-model-llama3/meta.yaml b/script/get-ml-model-llama3/meta.yaml index 8b26d7fff..5e2c04fc1 100644 --- a/script/get-ml-model-llama3/meta.yaml +++ b/script/get-ml-model-llama3/meta.yaml @@ -59,6 +59,27 @@ variations: env: MLC_ML_MODEL_NAME: Llama-3.1-8b-Instruct MLC_ML_MODEL_R2_HOSTED_NAME: llama3-1-8b-instruct + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_mlperf-llama3-1 + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + default: true + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader mlc: group: download-src default: true @@ -92,27 +113,6 @@ variations: adr: dae: extra_cache_tags: llama3,dataset,rclone - rclone: - group: download-tool - add_deps_recursive: - dae: - tags: _rclone - prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_mlperf-llama3-1 - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - default: true - r2-downloader: - group: download-tool - add_deps_recursive: - dae: - tags: _r2-downloader dry-run: group: run-mode env: diff --git a/script/get-ml-model-pointpainting/customize.py b/script/get-ml-model-pointpainting/customize.py index 07b18602d..ec30f0d2b 100644 --- a/script/get-ml-model-pointpainting/customize.py +++ b/script/get-ml-model-pointpainting/customize.py @@ -24,13 +24,6 @@ def preprocess(i): if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') == '' or env.get( 'MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" - if env['MLC_DOWNLOAD_SRC'] == "mlcommons": - i['run_script_input']['script_name'] = 'run-rclone' - if env.get('MLC_OUTDIRNAME', '') != '': - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'] = env['MLC_OUTDIRNAME'] - else: - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'] = os.path.join( - os.getcwd(), "model") return {'return': 0} @@ -39,21 +32,22 @@ def postprocess(i): env = i['env'] - if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') == '': - if env['MLC_ML_MODEL_PP_FORMAT'] == "onnx": - env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp.onnx") - else: - env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp_ep36.pth") - - if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': - if env['MLC_ML_MODEL_DPLAB_RESNET50_FORMAT'] == "onnx": - env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "deeplabv3+.onnx") - else: - env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( - env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], - "best_deeplabv3plus_resnet50_waymo_os16.pth") + if env.get('MLC_DOWNLOAD_MODE', '') != "dry": + if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') == '': + if env['MLC_ML_MODEL_PP_FORMAT'] == "onnx": + env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp.onnx") + else: + env['MLC_ML_MODEL_POINT_PAINTING_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "pp_ep36.pth") + + if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') == '': + if env['MLC_ML_MODEL_DPLAB_RESNET50_FORMAT'] == "onnx": + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], "deeplabv3+.onnx") + else: + env['MLC_ML_MODEL_DPLAB_RESNET50_PATH'] = os.path.join( + env['MLC_ML_MODEL_POINT_PAINTING_TMP_PATH'], + "best_deeplabv3plus_resnet50_waymo_os16.pth") return {'return': 0} diff --git a/script/get-ml-model-pointpainting/meta.yaml b/script/get-ml-model-pointpainting/meta.yaml index b811d58d6..8047b6188 100644 --- a/script/get-ml-model-pointpainting/meta.yaml +++ b/script/get-ml-model-pointpainting/meta.yaml @@ -27,9 +27,11 @@ variations: env: MLC_ML_MODEL_PP_FORMAT: onnx MLC_ML_MODEL_DPLAB_RESNET50_FORMAT: onnx - mlc: - group: download-src - default: true + rclone: + group: download-tool + add_deps_recursive: + dae: + tags: _rclone prehook_deps: - tags: get,rclone enable_if_env: @@ -40,6 +42,47 @@ variations: enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - yes + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader + default: true + mlc: + group: download-src + default: true + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_POINT_PAINTING_TMP_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_POINT_PAINTING_TMP_PATH + extra_cache_tags: waymo,dataset + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL env: MLC_DOWNLOAD_SRC: mlcommons - + dry-run: + group: run-mode + env: + MLC_DOWNLOAD_MODE: dry + dry-run,rclone: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + rclone,mlc: + env: + MLC_DOWNLOAD_URL: mlc-waymo:waymo_preprocessed_dataset/model + r2-downloader,mlc: + env: + MLC_DOWNLOAD_URL: https://waymo.mlcommons-storage.org/metadata/model.uri diff --git a/script/get-ml-model-pointpainting/run-rclone.sh b/script/get-ml-model-pointpainting/run-rclone.sh deleted file mode 100644 index 9b76a1511..000000000 --- a/script/get-ml-model-pointpainting/run-rclone.sh +++ /dev/null @@ -1,4 +0,0 @@ -cmd="rclone sync mlc-waymo:waymo_preprocessed_dataset/model ${MLC_ML_MODEL_POINT_PAINTING_TMP_PATH} -P" -echo $cmd -eval $cmd -test $? -eq 0 || exit $? \ No newline at end of file From 11cf15dc1f9511bde57214dfd44c0e9b5e24d67d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 29 Jul 2025 16:14:27 +0530 Subject: [PATCH 051/124] Update customize.py --- script/download-file/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 3e0665c79..6a2eb01e9 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -203,7 +203,7 @@ def preprocess(i): env['MLC_DOWNLOAD_CMD'] += f" || (({del_cmd} {env['MLC_DOWNLOAD_FILENAME']} || true) && wget -nc {extra_download_options} {url})" logger.info(f"{env['MLC_DOWNLOAD_CMD']}") - elif tool == "r2_downloader": + elif tool == "r2-downloader": env['MLC_DOWNLOAD_CMD'] = f"bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) " if env["MLC_HOST_OS_TYPE"] == "windows": # have to modify the variable from url to temp_url if it is From 3d79cb88f364c0a335bcab0557c319b699d79c9a Mon Sep 17 00:00:00 2001 From: Thaw Zin Htoo <112410262+kamieyy@users.noreply.github.com> Date: Tue, 29 Jul 2025 23:20:36 +0700 Subject: [PATCH 052/124] Updated meta.yaml with requested changes --- script/get-ml-model-whisper/meta.yaml | 64 ++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index c83079ae7..3c15391fe 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -7,13 +7,12 @@ env: MLC_ML_MODEL_WEIGHT_TRANSFORMATIONS: 'no' MLC_ML_MODEL_DATASET: whisper input_mapping: - checkpoint: WHISPER_CHECKPOINT_PATH + checkpoint: MLC_ML_MODEL_WHISPER_PATH new_env_keys: -- MLC_ML_MODEL_* -- WHISPER_CHECKPOINT_PATH -- MLC_WHISPER_FINAL_SAFE_TENSORS_PATH +- MLC_ML_MODEL_WHISPER_PATH +- MLC_ML_MODEL_FILE_WITH_PATH print_env_at_the_end: - WHISPER_CHECKPOINT_PATH: Whisper checkpoint path + MLC_ML_MODEL_WHISPER_PATH: Whisper checkpoint path tags: - get-ml-model-whisper - get @@ -23,7 +22,9 @@ tags: tests: run_inputs: - variations_list: - - huggingface,dry-run + - rclone,mlc,dry-run + - r2-downloader,mlc,dry-run + - huggingface uid: 3bea2356e97f47b1 @@ -84,4 +85,53 @@ variations: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run dry-run,r2-downloader: env: - MLC_DOWNLOAD_EXTRA_OPTIONS: -x \ No newline at end of file + MLC_DOWNLOAD_EXTRA_OPTIONS: -x + mlc: + default: true + env: + MLC_DOWNLOAD_SRC: mlcommons + group: download-src + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_ML_MODEL_WHISPER_PATH + MLC_EXTRACT_FINAL_ENV_NAME: MLC_ML_MODEL_WHISPER_PATH + extra_cache_tags: ml,model,whisper + force_cache: true + force_env_keys: + - MLC_OUTDIRNAME + names: + - whisper-model-dae + tags: download-and-extract + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL + rclone: + adr: + whisper-model-dae: + tags: _rclone + env: + MLC_DOWNLOAD_URL: 'mlc-inference:mlcommons-inference-wg-public/Whisper/model/' + prehook_deps: + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + tags: get,rclone + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC + force_cache: true + tags: get,rclone-config,_mlc-inference + default: true + group: download-tool + r2-downloader: + adr: + whisper-model-dae: + tags: _r2-downloader + env: + MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' + group: download-tool From 179aaa0139759e63e13545c9525125903415bfc4 Mon Sep 17 00:00:00 2001 From: kamieyy Date: Tue, 29 Jul 2025 23:28:54 +0700 Subject: [PATCH 053/124] removed run.sh script --- script/get-ml-model-whisper/run.sh | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 script/get-ml-model-whisper/run.sh diff --git a/script/get-ml-model-whisper/run.sh b/script/get-ml-model-whisper/run.sh deleted file mode 100644 index f0e290b83..000000000 --- a/script/get-ml-model-whisper/run.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e - -# Default checkpoint path -CHECKPOINT_PATH=${MLC_ML_MODEL_WHISPER_PATH:-whisper-large-v3} - -git lfs install - -if [ ! -d "$CHECKPOINT_PATH" ]; then - git clone https://huggingface.co/openai/whisper-large-v3 "$CHECKPOINT_PATH" -fi - -cd "${CHECKPOINT_PATH}" -git checkout 06f233fe06e710322aca913c1bc4249a0d71fce1 From 8b01f4c4efbcfff109d014621e42d878062f254a Mon Sep 17 00:00:00 2001 From: kamieyy Date: Tue, 29 Jul 2025 23:41:54 +0700 Subject: [PATCH 054/124] added validation for MLC_ML_MODEL_WHISPER_PATH in customize.py --- script/get-ml-model-whisper/customize.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/script/get-ml-model-whisper/customize.py b/script/get-ml-model-whisper/customize.py index 1a1ef7555..a0f46ab85 100644 --- a/script/get-ml-model-whisper/customize.py +++ b/script/get-ml-model-whisper/customize.py @@ -12,8 +12,26 @@ def preprocess(i): if os_info['platform'] == "windows": return {'return': 1, 'error': 'Script not supported in windows yet!'} - if env.get('MLC_ML_MODEL_WHISPER_PATH', '') == '': + checkpoint_path = env.get('MLC_ML_MODEL_WHISPER_PATH', '').strip() + + if checkpoint_path == '': env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes" + else: + # Normalize and expand the path + checkpoint_path = os.path.abspath(os.path.expanduser(checkpoint_path)) + env['MLC_ML_MODEL_WHISPER_PATH'] = checkpoint_path + + if not os.path.exists(checkpoint_path): + return { + 'return': 1, + 'error': f"Provided Whisper model path '{checkpoint_path}' does not exist." + } + + if not os.path.isdir(checkpoint_path): + return { + 'return': 1, + 'error': f"Provided Whisper model path '{checkpoint_path}' is not a directory." + } return {'return': 0} From 4934283ddfa8a798b41fcb075c301f5ebe6e2db9 Mon Sep 17 00:00:00 2001 From: kamieyy Date: Tue, 29 Jul 2025 23:42:12 +0700 Subject: [PATCH 055/124] fixed indentation issues --- script/get-ml-model-whisper/meta.yaml | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index 3c15391fe..cf48cbb9e 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -41,8 +41,8 @@ prehook_deps: names: - hf-zoo tags: get,ml-model,huggingface,zoo,_clone-repo - force_env_keys: - - MLC_OUTDIRNAME +- force_env_keys: + - MLC_OUTDIRNAME variations: fp32: @@ -54,7 +54,7 @@ variations: group: precision huggingface: - group: download-source + group: download-src env: MLC_DOWNLOAD_SRC: huggingface @@ -115,17 +115,17 @@ variations: env: MLC_DOWNLOAD_URL: 'mlc-inference:mlcommons-inference-wg-public/Whisper/model/' prehook_deps: - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - tags: get,rclone - - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - true - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC - force_cache: true - tags: get,rclone-config,_mlc-inference + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + tags: get,rclone + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - true + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 17CpM5eU8tjrxh_LpH_BTNTeT37PhzcnC + force_cache: true + tags: get,rclone-config,_mlc-inference default: true group: download-tool r2-downloader: From e985c6bf3176d7b3cfb8e0b59279976a840baa0c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 29 Jul 2025 18:47:36 +0100 Subject: [PATCH 056/124] Update customize.py | Fix typo --- script/app-mlperf-inference-mlcommons-python/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index c29dd26d2..37db887c5 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -609,7 +609,7 @@ def get_run_cmd_reference( if env.get('MLC_MLPERF_POINTPAINTING_TIME', '') != '': cmd += f" --time {env['MLC_MLPERF_POINTPAINTING_TIME']}" - logger.info(fcmd) + logger.info(cmd) elif "deepseek-r1" in env['MLC_MODEL']: env['RUN_DIR'] = os.path.join( From 48b0df34d9ba85014e9592524bec844628ec161e Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Jul 2025 09:36:19 +0530 Subject: [PATCH 057/124] llama3.1 -> llama3_1 (#554) --- script/app-mlperf-inference/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index cfcb5ef1a..871ca8d24 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -1012,7 +1012,7 @@ variations: MLC_USE_DATASET_FROM_HOST: - 'yes' - llama3.1-8b-edge: + llama3_1-8b-edge: base: - llama3_1-8b_ add_deps_recursive: From 943f23a50bd8448b4427be5ec5615ad5b1897bc0 Mon Sep 17 00:00:00 2001 From: Thaw Zin Htoo <112410262+kamieyy@users.noreply.github.com> Date: Wed, 30 Jul 2025 12:04:56 +0700 Subject: [PATCH 058/124] added missing tags field to prehook_deps entry The automation framework automation/script/module.py dep_tags_list = dep.get('tags').split(",") expects every dependency to have a tags string, otherwise it throws an error. --- script/get-ml-model-whisper/meta.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index cf48cbb9e..b4f128bcd 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -43,7 +43,11 @@ prehook_deps: tags: get,ml-model,huggingface,zoo,_clone-repo - force_env_keys: - MLC_OUTDIRNAME + names: + - whisper-outdir-setup + tags: setup,ml-model + variations: fp32: default: true From 947fb59fd9353b53da10e8a893a889f0515394cf Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 30 Jul 2025 14:43:27 +0530 Subject: [PATCH 059/124] Fix Waymo dataset extraction (#555) --- script/get-dataset-waymo/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-dataset-waymo/run.sh b/script/get-dataset-waymo/run.sh index fa7001aad..0af8d4a79 100644 --- a/script/get-dataset-waymo/run.sh +++ b/script/get-dataset-waymo/run.sh @@ -7,8 +7,8 @@ #${MLC_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency -if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "true" ]]; then - cd "${MLC_DATASET_WAYMO_PATH}/kitti_format/training" || exit +if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then + cd "${MLC_DATASET_WAYMO_PATH}/training" || exit for f in *.tar.gz; do tar -xzvf "$f"; done cd - || exit fi \ No newline at end of file From 903bf7184d030004ca7f394b008660bf11f50282 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 30 Jul 2025 11:26:59 +0100 Subject: [PATCH 060/124] Fixes for mlperf inference power --- script/benchmark-program-mlperf/customize.py | 2 +- script/run-all-mlperf-models/run-mobilenet-models.sh | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/script/benchmark-program-mlperf/customize.py b/script/benchmark-program-mlperf/customize.py index bb74f7f4c..8bc500afd 100644 --- a/script/benchmark-program-mlperf/customize.py +++ b/script/benchmark-program-mlperf/customize.py @@ -1,5 +1,5 @@ from mlc import utils -from utils import is_true +from utils import is_true,is_false import os diff --git a/script/run-all-mlperf-models/run-mobilenet-models.sh b/script/run-all-mlperf-models/run-mobilenet-models.sh index 557332d93..f0500c418 100644 --- a/script/run-all-mlperf-models/run-mobilenet-models.sh +++ b/script/run-all-mlperf-models/run-mobilenet-models.sh @@ -22,8 +22,8 @@ function run() { exit_if_error fi } -POWER=" --power=yes --adr.mlperf-power-client.power_server=192.168.0.15 --adr.mlperf-power-client.port=4940 --adr.mlperf-power-client.ntp_server=time.google.com " -POWER="" +POWER=" --power=yes --adr.mlperf-power-client.power_server=192.168.1.79 --adr.mlperf-power-client.port=4950 --adr.mlperf-power-client.ntp_server=time.google.com " +#POWER="" #extra_option=" --minimize_disk_usage=yes" extra_option=" --minimize_disk_usage=no" extra_tags="" @@ -35,11 +35,8 @@ extra_tags="" # run "$MLC_RUN_CMD" run "mlcr run,mobilenet-models,_tflite$extra_tags \ --adr.compiler.tags=gcc \ -${extra_option} " +${extra_option} $POWER" -run "mlcr run,mobilenet-models,_tflite,_armnn,_neon$extra_tags \ ---adr.compiler.tags=gcc \ -${extra_option} " extra_option=" $extra_option --adr.mlperf-inference-implementation.compressed_dataset=on" From e7dbf00655bd7f019f8efc74bc5479cacdf8b5a9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 30 Jul 2025 10:27:23 +0000 Subject: [PATCH 061/124] [Automated Commit] Format Codebase [skip ci] --- script/benchmark-program-mlperf/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/benchmark-program-mlperf/customize.py b/script/benchmark-program-mlperf/customize.py index 8bc500afd..9172a690f 100644 --- a/script/benchmark-program-mlperf/customize.py +++ b/script/benchmark-program-mlperf/customize.py @@ -1,5 +1,5 @@ from mlc import utils -from utils import is_true,is_false +from utils import is_true, is_false import os From ffcf5ae61de8a0090b19b9b92d07e6565b0b6b1b Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 30 Jul 2025 12:21:31 +0100 Subject: [PATCH 062/124] Fixes the CLI upload of mlperf inference results --- .../customize.py | 3 ++ script/submit-mlperf-results/customize.py | 48 +++++++++++-------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py index 91b0d94c1..b8d4000ba 100644 --- a/script/run-mlperf-inference-submission-checker/customize.py +++ b/script/run-mlperf-inference-submission-checker/customize.py @@ -65,6 +65,9 @@ def preprocess(i): extra_args = ' ' + env.get('MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS', '') + if (is_true(env.get('MLC_TAR_SUBMISSION_DIR')) or env.get('MLC_MLPERF_SUBMITTER_ID', '') != '') and "skip-extra-files-in-root-check" not in extra_args: + extra_args += " --skip-extra-files-in-root-check " + x_submitter = ' --submitter ' + q + submitter + q if submitter != '' else '' x_version = ' --version ' + version + ' ' if version != '' else '' diff --git a/script/submit-mlperf-results/customize.py b/script/submit-mlperf-results/customize.py index 1c2c6b619..7b1a17015 100644 --- a/script/submit-mlperf-results/customize.py +++ b/script/submit-mlperf-results/customize.py @@ -2,7 +2,8 @@ from mlc import utils import os import json - +import shutil +import tempfile def preprocess(i): @@ -25,26 +26,31 @@ def preprocess(i): env['MLC_TMP_CURRENT_PATH'], file_path)) - r = get_signed_url( - server, - benchmark, - submitter_id, - submitter_name, - file_path) - if r['return'] > 0: - return r - - signed_url = r['signed_url'] - submission_id = r['submission_id'] - - r = upload_file_to_signed_url(file_path, signed_url, logger) - if r['return'] > 0: - return r - - r = trigger_submission_checker( - server, submitter_id, benchmark, submission_id, logger) - if r['return'] > 0: - return r + with tempfile.TemporaryDirectory() as tmpdir: + tmp_file_path = os.path.join(tmpdir, "submission.tar.gz") + + shutil.copy(file_path, tmp_file_path) #use tmp_file_path to prevent long file paths which causes issue during upload + + r = get_signed_url( + server, + benchmark, + submitter_id, + submitter_name, + tmp_file_path) + if r['return'] > 0: + return r + + signed_url = r['signed_url'] + submission_id = r['submission_id'] + + r = upload_file_to_signed_url(tmp_file_path, signed_url, logger) + if r['return'] > 0: + return r + + r = trigger_submission_checker( + server, submitter_id, benchmark, submission_id, logger) + if r['return'] > 0: + return r return {'return': 0} From 86c36370958ffaf0a1294ffbe73192853632afb1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 30 Jul 2025 11:21:52 +0000 Subject: [PATCH 063/124] [Automated Commit] Format Codebase [skip ci] --- script/run-mlperf-inference-submission-checker/customize.py | 3 ++- script/submit-mlperf-results/customize.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/script/run-mlperf-inference-submission-checker/customize.py b/script/run-mlperf-inference-submission-checker/customize.py index b8d4000ba..a427431e9 100644 --- a/script/run-mlperf-inference-submission-checker/customize.py +++ b/script/run-mlperf-inference-submission-checker/customize.py @@ -65,7 +65,8 @@ def preprocess(i): extra_args = ' ' + env.get('MLC_MLPERF_SUBMISSION_CHECKER_EXTRA_ARGS', '') - if (is_true(env.get('MLC_TAR_SUBMISSION_DIR')) or env.get('MLC_MLPERF_SUBMITTER_ID', '') != '') and "skip-extra-files-in-root-check" not in extra_args: + if (is_true(env.get('MLC_TAR_SUBMISSION_DIR')) or env.get('MLC_MLPERF_SUBMITTER_ID', + '') != '') and "skip-extra-files-in-root-check" not in extra_args: extra_args += " --skip-extra-files-in-root-check " x_submitter = ' --submitter ' + q + submitter + q if submitter != '' else '' diff --git a/script/submit-mlperf-results/customize.py b/script/submit-mlperf-results/customize.py index 7b1a17015..b6ec51754 100644 --- a/script/submit-mlperf-results/customize.py +++ b/script/submit-mlperf-results/customize.py @@ -5,6 +5,7 @@ import shutil import tempfile + def preprocess(i): os_info = i['os_info'] @@ -29,7 +30,9 @@ def preprocess(i): with tempfile.TemporaryDirectory() as tmpdir: tmp_file_path = os.path.join(tmpdir, "submission.tar.gz") - shutil.copy(file_path, tmp_file_path) #use tmp_file_path to prevent long file paths which causes issue during upload + # use tmp_file_path to prevent long file paths which causes issue + # during upload + shutil.copy(file_path, tmp_file_path) r = get_signed_url( server, From 95e17405150fe97db61d833b15e66d52e77ef40c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 30 Jul 2025 20:35:01 +0100 Subject: [PATCH 064/124] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 86b93ac06..55e79c0ae 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -5,6 +5,7 @@ on: types: [published] workflow_dispatch: {} + jobs: build_wheels: if: github.repository_owner == 'mlcommons' From 82cd4d77c9d362e147c0867afeeb660423290ebe Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 30 Jul 2025 21:42:51 +0000 Subject: [PATCH 065/124] fixes for llama3.1 mlperf inference run --- script/app-mlperf-inference-mlcommons-python/customize.py | 2 +- script/app-mlperf-inference/meta.yaml | 2 ++ script/process-mlperf-accuracy/meta.yaml | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index 37db887c5..ae19507ad 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -537,7 +537,7 @@ def get_run_cmd_reference( --tensor-parallel-size {env['MLC_MLPERF_INFERENCE_TP_SIZE']} \ {env['MLC_MLPERF_LOADGEN_EXTRA_OPTIONS']} \ {scenario_extra_options} {mode_extra_options} \ - --vllm""" + --vllm --lg-model-name {env['MLC_MODEL']}""" if env.get('MLC_MLPERF_INFERENCE_NUM_WORKERS', '') != '': cmd += f" --num-workers {env['MLC_MLPERF_INFERENCE_NUM_WORKERS']}" diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 871ca8d24..522c42148 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -223,6 +223,8 @@ variations: tags: _int32 llama3_1-405b-accuracy-script: tags: _int32 + llama3_1-8b-accuracy-script: + tags: _int32 env: MLC_MLPERF_PYTHON: 'yes' MLC_MLPERF_IMPLEMENTATION: mlcommons_python diff --git a/script/process-mlperf-accuracy/meta.yaml b/script/process-mlperf-accuracy/meta.yaml index 8f25f68fb..52a7dcd14 100644 --- a/script/process-mlperf-accuracy/meta.yaml +++ b/script/process-mlperf-accuracy/meta.yaml @@ -353,7 +353,7 @@ variations: group: dataset cnndm_llama_3,edge: deps: - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_rclone + - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-edge skip_if_env: @@ -361,7 +361,7 @@ variations: - "yes" cnndm_llama_3,datacenter: deps: - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_rclone + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader names: - cnndm-llama3-datacenter skip_if_env: From ba3f0c4aded144c551c2a82616fb4f4662cd0eab Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Jul 2025 13:26:36 +0530 Subject: [PATCH 066/124] Fix typo --- script/get-ml-model-pointpainting/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-ml-model-pointpainting/customize.py b/script/get-ml-model-pointpainting/customize.py index ec30f0d2b..7e617fca5 100644 --- a/script/get-ml-model-pointpainting/customize.py +++ b/script/get-ml-model-pointpainting/customize.py @@ -12,9 +12,9 @@ def preprocess(i): return {'return': 1, 'error': 'Script not supported in windows yet!'} if env.get('MLC_ML_MODEL_POINT_PAINTING_PATH', '') != '': - if not os.path.exists(env['MLC_ML_MODEL_POINT_PAINTING']): + if not os.path.exists(env['MLC_ML_MODEL_POINT_PAINTING_PATH']): return { - 'return': 1, 'error': f"Provided model path {env['MLC_ML_MODEL_POINT_PAINTING']} does not exist."} + 'return': 1, 'error': f"Provided model path {env['MLC_ML_MODEL_POINT_PAINTING_PATH']} does not exist."} if env.get('MLC_ML_MODEL_DPLAB_RESNET50_PATH', '') != '': if not os.path.exists(env['MLC_ML_MODEL_DPLAB_RESNET50_PATH']): From 1dbfa8fc3d89e5022f85798b680c954f1492d1b0 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 31 Jul 2025 13:41:10 +0530 Subject: [PATCH 067/124] Fix typo --- script/app-mlperf-inference/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 522c42148..61b674cdb 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -939,7 +939,7 @@ variations: - waymo-dataset - tags: get,ml-model,pointpainting enable_if_env: - MLC_USE_DATASET_FROM_HOST: + MLC_USE_ML_MODEL_FROM_HOST: - 'yes' names: - pointpainting-model From 0e887401a8faecc661958c0201d54d8298c224e8 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 1 Aug 2025 00:48:47 +0530 Subject: [PATCH 068/124] precheck required files before generating submission --- .../customize.py | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index f6490a31c..6a0022f75 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -401,11 +401,46 @@ def generate_submission(env, state, inp, submission_division, logger): # we check for the existance of mlperf_log_summary.txt # mlperf_log_detail.txt to consider a result folder as valid. # Rest of the checks are done later by the submission checker - files_to_check = [ + files_to_check_in_perf_dir = [ "mlperf_log_summary.txt", "mlperf_log_detail.txt"] - if not all([os.path.exists(os.path.join( - result_scenario_path, "performance", "run_1", f)) for f in files_to_check]): + perf_run_dir = os.path.join( + result_scenario_path, "performance", "run_1") + missing = [ + f for f in files_to_check_in_perf_dir + if not os.path.exists(os.path.join(perf_run_dir, f)) + ] + if missing: + logger.warning( + f"Missing file(s) in {perf_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}") + continue + + files_to_check_in_acc_dir = [ + "mlperf_log_summary.txt", + "mlperf_log_detail.txt", + "mlperf_log_accuracy.json", + "accuracy.txt"] + acc_run_dir = os.path.join( + result_scenario_path, "accuracy") + missing = [ + f for f in files_to_check_in_acc_dir + if not os.path.exists(os.path.join(acc_run_dir, f)) + ] + if missing: + logger.warning( + f"Missing file(s) in {acc_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}") + continue + + if not os.path.exists(os.path.join(perf_run_dir, "user.conf")) and not os.path.exists(os.path.join(result_scenario_path, "user.conf")): + logger.warning( + f"Missing user.conf in both {os.path.join(perf_run_dir, "user.conf")} and {os.path.join(result_scenario_path, "user.conf")}. Skipping directory: {result_scenario_path}" + ) + continue + + if not os.path.exists(os.path.join(perf_run_dir, "measurements.json")) and not os.path.exists(os.path.join(result_scenario_path, "measurements.json")): + logger.warning( + f"Missing measurements.json in both {os.path.join(perf_run_dir, "measurements.json")} and {os.path.join(result_scenario_path, "measurements.json")}. Skipping directory: {result_scenario_path}" + ) continue if not os.path.isdir(measurement_scenario_path): From b7da7fa74d97bad16091bf7bc144d9bc630a16ed Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 1 Aug 2025 01:15:21 +0530 Subject: [PATCH 069/124] fix f-string error --- script/generate-mlperf-inference-submission/customize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 6a0022f75..4c4770bd5 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -412,7 +412,7 @@ def generate_submission(env, state, inp, submission_division, logger): ] if missing: logger.warning( - f"Missing file(s) in {perf_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}") + f"""Missing file(s) in {perf_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}""") continue files_to_check_in_acc_dir = [ @@ -428,18 +428,18 @@ def generate_submission(env, state, inp, submission_division, logger): ] if missing: logger.warning( - f"Missing file(s) in {acc_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}") + f"""Missing file(s) in {acc_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}""") continue if not os.path.exists(os.path.join(perf_run_dir, "user.conf")) and not os.path.exists(os.path.join(result_scenario_path, "user.conf")): logger.warning( - f"Missing user.conf in both {os.path.join(perf_run_dir, "user.conf")} and {os.path.join(result_scenario_path, "user.conf")}. Skipping directory: {result_scenario_path}" + f"""Missing user.conf in both {os.path.join(perf_run_dir, "user.conf")} and {os.path.join(result_scenario_path, "user.conf")}. Skipping directory: {result_scenario_path}""" ) continue if not os.path.exists(os.path.join(perf_run_dir, "measurements.json")) and not os.path.exists(os.path.join(result_scenario_path, "measurements.json")): logger.warning( - f"Missing measurements.json in both {os.path.join(perf_run_dir, "measurements.json")} and {os.path.join(result_scenario_path, "measurements.json")}. Skipping directory: {result_scenario_path}" + f"""Missing measurements.json in both {os.path.join(perf_run_dir, "measurements.json")} and {os.path.join(result_scenario_path, "measurements.json")}. Skipping directory: {result_scenario_path}""" ) continue From 231baa6884eba94fa1d5c71bc967d8d7e981d7a5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 31 Jul 2025 19:45:37 +0000 Subject: [PATCH 070/124] [Automated Commit] Format Codebase [skip ci] --- .../customize.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/script/generate-mlperf-inference-submission/customize.py b/script/generate-mlperf-inference-submission/customize.py index 4c4770bd5..8ca22b596 100644 --- a/script/generate-mlperf-inference-submission/customize.py +++ b/script/generate-mlperf-inference-submission/customize.py @@ -405,7 +405,7 @@ def generate_submission(env, state, inp, submission_division, logger): "mlperf_log_summary.txt", "mlperf_log_detail.txt"] perf_run_dir = os.path.join( - result_scenario_path, "performance", "run_1") + result_scenario_path, "performance", "run_1") missing = [ f for f in files_to_check_in_perf_dir if not os.path.exists(os.path.join(perf_run_dir, f)) @@ -414,14 +414,14 @@ def generate_submission(env, state, inp, submission_division, logger): logger.warning( f"""Missing file(s) in {perf_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}""") continue - + files_to_check_in_acc_dir = [ "mlperf_log_summary.txt", "mlperf_log_detail.txt", "mlperf_log_accuracy.json", "accuracy.txt"] acc_run_dir = os.path.join( - result_scenario_path, "accuracy") + result_scenario_path, "accuracy") missing = [ f for f in files_to_check_in_acc_dir if not os.path.exists(os.path.join(acc_run_dir, f)) @@ -431,13 +431,15 @@ def generate_submission(env, state, inp, submission_division, logger): f"""Missing file(s) in {acc_run_dir}: {', '.join(missing)}, Skipping directory: {result_scenario_path}""") continue - if not os.path.exists(os.path.join(perf_run_dir, "user.conf")) and not os.path.exists(os.path.join(result_scenario_path, "user.conf")): + if not os.path.exists(os.path.join(perf_run_dir, "user.conf")) and not os.path.exists( + os.path.join(result_scenario_path, "user.conf")): logger.warning( f"""Missing user.conf in both {os.path.join(perf_run_dir, "user.conf")} and {os.path.join(result_scenario_path, "user.conf")}. Skipping directory: {result_scenario_path}""" ) continue - - if not os.path.exists(os.path.join(perf_run_dir, "measurements.json")) and not os.path.exists(os.path.join(result_scenario_path, "measurements.json")): + + if not os.path.exists(os.path.join(perf_run_dir, "measurements.json")) and not os.path.exists( + os.path.join(result_scenario_path, "measurements.json")): logger.warning( f"""Missing measurements.json in both {os.path.join(perf_run_dir, "measurements.json")} and {os.path.join(result_scenario_path, "measurements.json")}. Skipping directory: {result_scenario_path}""" ) From 869c06596429c690a6d9ef2c8222fc57e3459a0f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Aug 2025 01:16:52 +0530 Subject: [PATCH 071/124] Update build_wheel_off.yml --- .github/workflows/build_wheel_off.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheel_off.yml b/.github/workflows/build_wheel_off.yml index 4fbc24dba..9a7eb65e9 100644 --- a/.github/workflows/build_wheel_off.yml +++ b/.github/workflows/build_wheel_off.yml @@ -1,5 +1,6 @@ name: Build wheel and release into PYPI (off now) + on: push: branches: From 93eda2295624b1f39b4e9abec9d5f95c13f1902f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Fri, 1 Aug 2025 16:52:38 +0530 Subject: [PATCH 072/124] update llama3.1 8b edge dataset link - rclone --- script/get-dataset-cnndm/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index 1bdf3db37..bb7d05e44 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -131,7 +131,7 @@ variations: dae: extra_cache_tags: cnndm,dataset,llama3,val,edge env: - MLC_DATASET_CNNDM_FILENAME: sample_cnn_eval_5000.json + MLC_DATASET_CNNDM_FILENAME: cnn_eval_5000.json MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH From 9bf1f0faa372c8a19af337a0ca86f7f79ff604e6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 2 Aug 2025 03:43:17 +0530 Subject: [PATCH 073/124] Update meta.yaml --- script/get-dataset-cnndm/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index bb7d05e44..9286cda4e 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -132,7 +132,7 @@ variations: extra_cache_tags: cnndm,dataset,llama3,val,edge env: MLC_DATASET_CNNDM_FILENAME: cnn_eval_5000.json - MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> + MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/datasets/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH validation,edge,llama3,mlc,r2-downloader: @@ -150,7 +150,7 @@ variations: extra_cache_tags: cnndm,dataset,llama3,val,datacenter env: MLC_DATASET_CNNDM_FILENAME: cnn_eval.json - MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> + MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/datasets/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH validation,datacenter,llama3,mlc,r2-downloader: From c372385cc259ed35d9671e136e4c2059c4038500 Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 2 Aug 2025 22:05:58 +0000 Subject: [PATCH 074/124] Fixes cnndm dataset download --- .../meta.yaml | 7 +-- script/get-dataset-cnndm/meta.yaml | 53 ++++++++++++++----- script/process-mlperf-accuracy/meta.yaml | 4 +- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 7980fa9f1..13415df06 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -526,12 +526,13 @@ deps: - "yes" ## LLAMA3_1-8B - - tags: get,ml-model,llama3,_mlc,_8b,_r2-downloader + - tags: get,ml-model,llama3,_8b names: - llama3-8b-model enable_if_env: MLC_MODEL: - llama3_1-8b + - llama3_1-8b-edge skip_if_env: MLC_USE_MODEL_FROM_HOST: - "yes" @@ -1476,7 +1477,7 @@ variations: MLC_MODEL: llama3_1-8b deps: ## CNNDM for Llama3 8B model - datacenter - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3 names: - cnndm-llama3-datacenter skip_if_env: @@ -1490,7 +1491,7 @@ variations: MLC_MODEL: llama3_1-8b-edge deps: ## CNNDM for Llama3 8B model - edge - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader + - tags: get,dataset,cnndm,_validation,_edge,_llama3 names: - cnndm-llama3-edge skip_if_env: diff --git a/script/get-dataset-cnndm/meta.yaml b/script/get-dataset-cnndm/meta.yaml index 1bdf3db37..eba3109c7 100644 --- a/script/get-dataset-cnndm/meta.yaml +++ b/script/get-dataset-cnndm/meta.yaml @@ -39,6 +39,34 @@ deps: skip_if_env: MLC_TMP_ML_MODEL: - llama3_1-8b +prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_CNNDM_DOWNLOAD_TOOL: + - rclone + - tags: get,rclone-config,_mlc-inference + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + MLC_CNNDM_DOWNLOAD_TOOL: + - rclone + - enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - 'yes' + env: + MLC_RCLONE_COPY_USING: copy + force_cache: true + names: + - dae + tags: download-and-extract + force_env_keys: + - MLC_OUTDIRNAME + update_tags_from_env_with_prefix: + _url.: + - MLC_DOWNLOAD_URL env: MLC_DATASET: CNNDM tags: @@ -58,21 +86,15 @@ variations: datacenter: group: category rclone: - prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_mlc-inference - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes + env: + MLC_CNNDM_DOWNLOAD_TOOL: rclone group: download-tool add_deps_recursive: dae: tags: _rclone r2-downloader: + env: + MLC_CNNDM_DOWNLOAD_TOOL: r2-downloader group: download-tool add_deps_recursive: dae: @@ -126,13 +148,18 @@ variations: - MLC_DATASET_PATH - MLC_DATASET_EVAL_PATH - MLC_DATASET_CNNDM_EVAL_PATH + validation,llama3: + default-variations: + download-tool: r2-downloader + download-src: mlc validation,edge,llama3,mlc,rclone: adr: dae: extra_cache_tags: cnndm,dataset,llama3,val,edge env: - MLC_DATASET_CNNDM_FILENAME: sample_cnn_eval_5000.json - MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<>> + MLC_DATASET_CNNDM_FILENAME: cnn_eval_5000.json + MLC_TMP_REQUIRE_DOWNLOAD: yes + MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/datasets/<<>> MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH validation,edge,llama3,mlc,r2-downloader: @@ -140,7 +167,7 @@ variations: dae: extra_cache_tags: cnndm,dataset,llama3,val,edge env: - MLC_DATASET_CNNDM_FILENAME: sample_cnn_eval_5000.json + MLC_DATASET_CNNDM_FILENAME: cnn_eval_5000.json MLC_DOWNLOAD_URL: https://inference.mlcommons-storage.org/metadata/llama3-1-8b-sample-cnn-eval-5000.uri MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH diff --git a/script/process-mlperf-accuracy/meta.yaml b/script/process-mlperf-accuracy/meta.yaml index 52a7dcd14..24ff6fdd0 100644 --- a/script/process-mlperf-accuracy/meta.yaml +++ b/script/process-mlperf-accuracy/meta.yaml @@ -353,7 +353,7 @@ variations: group: dataset cnndm_llama_3,edge: deps: - - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_r2-downloader + - tags: get,dataset,cnndm,_validation,_edge,_llama3 names: - cnndm-llama3-edge skip_if_env: @@ -361,7 +361,7 @@ variations: - "yes" cnndm_llama_3,datacenter: deps: - - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_r2-downloader + - tags: get,dataset,cnndm,_validation,_datacenter,_llama3 names: - cnndm-llama3-datacenter skip_if_env: From 4eb9d403875ab50d18a312d7a4da88b5d03e431a Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 2 Aug 2025 22:10:03 +0000 Subject: [PATCH 075/124] Fix defaults in preprocess-mlperf-inference-submission --- script/preprocess-mlperf-inference-submission/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/preprocess-mlperf-inference-submission/meta.yaml b/script/preprocess-mlperf-inference-submission/meta.yaml index f0b3c105a..81abf1a6e 100644 --- a/script/preprocess-mlperf-inference-submission/meta.yaml +++ b/script/preprocess-mlperf-inference-submission/meta.yaml @@ -38,8 +38,8 @@ input_mapping: submitter: MLC_MLPERF_SUBMITTER submission_preprocessor_args: MLC_MLPERF_PREPROCESS_SUBMISSION_EXTRA_ARGS default_env: - MLC_MLPERF_NOINFER_LOW_ACCURACY_RESULTS: True - MLC_MLPERF_NOINFER_SCENARIO_RESULTS: True + MLC_MLPERF_NOINFER_LOW_ACCURACY_RESULTS: False + MLC_MLPERF_NOINFER_SCENARIO_RESULTS: False tags: - run - mlc From 3042f8ce11127367f623098ef79cfa67950ba6d3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Aug 2025 21:44:53 +0530 Subject: [PATCH 076/124] Automatically mount --outdirname for docker run --- automation/script/docker.py | 3 ++- automation/script/docker_utils.py | 17 ++++++----------- automation/script/module.py | 27 ++++++++++++++------------- script/install-llvm-src/meta.yaml | 4 ++++ 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index 0d33391e7..dea7f3fb8 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -393,8 +393,9 @@ def docker_run(self_module, i): docker_inputs['mounts'] = res['mounts'] container_env_string = res['container_env_string'] + res = update_docker_environment( - docker_settings, env, container_env_string) + docker_settings, env, self_module.host_env_keys, container_env_string) if res['return'] > 0: return res diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index cb733f93a..ca7ab2af6 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -40,6 +40,10 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd, run_state): if 'mounts' in docker_settings: mounts.extend(docker_settings['mounts']) + for key in [ "MLC_INPUT", "MLC_OUTPUT", "MLC_OUTDIRNAME" ]: + if "${{ "+key+" }}:${{ "+key+" }}" not in mounts: + mounts.append("${{ "+key+" }}:${{ "+key+" }}") + docker_input_mapping = docker_settings.get("input_mapping", {}) container_env_string = "" @@ -182,7 +186,7 @@ def prepare_docker_inputs(input_params, docker_settings, return docker_inputs, dockerfile_path -def update_docker_environment(docker_settings, env, container_env_string): +def update_docker_environment(docker_settings, env, host_env_keys, container_env_string): """ Updates the Docker environment variables and build arguments. @@ -194,15 +198,6 @@ def update_docker_environment(docker_settings, env, container_env_string): Returns: dict: A dictionary with a return code indicating success or failure. """ - # Define proxy-related environment variable keys to propagate - proxy_keys = [ - "ftp_proxy", "FTP_PROXY", - "http_proxy", "HTTP_PROXY", - "https_proxy", "HTTPS_PROXY", - "no_proxy", "NO_PROXY", - "socks_proxy", "SOCKS_PROXY", - "GH_TOKEN" - ] # Ensure the '+ CM_DOCKER_BUILD_ARGS' key exists in the environment if '+ MLC_DOCKER_BUILD_ARGS' not in env: @@ -210,7 +205,7 @@ def update_docker_environment(docker_settings, env, container_env_string): # Add proxy environment variables to Docker build arguments and container # environment string - for proxy_key in proxy_keys: + for proxy_key in host_env_keys: proxy_value = os.environ.get(proxy_key) if proxy_value: container_env_string += f" --env.{proxy_key}={proxy_value} " diff --git a/automation/script/module.py b/automation/script/module.py index a2ab27753..9c2a3a3ea 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -67,6 +67,19 @@ def __init__(self, action_object, automation_file): 'MLC_GIT_*', 'MLC_RENEW_CACHE_ENTRY'] + self.host_env_keys = [ + "GH_TOKEN", + "ftp_proxy", + "FTP_PROXY", + "http_proxy", + "HTTP_PROXY", + "https_proxy", + "HTTPS_PROXY", + "no_proxy", + "NO_PROXY", + "socks_proxy", + "SOCKS_PROXY"] + self.input_flags_converted_to_tmp_env = { 'path': {'desc': 'Filesystem path to search for executable', 'default': ''}} @@ -486,19 +499,7 @@ def _run(self, i): 'append_unique': True}) # take some env from the user environment - keys = [ - "GH_TOKEN", - "ftp_proxy", - "FTP_PROXY", - "http_proxy", - "HTTP_PROXY", - "https_proxy", - "HTTPS_PROXY", - "no_proxy", - "NO_PROXY", - "socks_proxy", - "SOCKS_PROXY"] - for key in keys: + for key in self.host_env_keys: if os.environ.get(key, '') != '' and env.get(key, '') == '': env[key] = os.environ[key] diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index c59f0a5c2..ca44d5003 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -247,6 +247,10 @@ variations: - full-history env: MLC_GIT_CHECKOUT_TAG: '#' +docker: + skip_run_cmd: true + pre_run_cmds: + - mlc pull repo versions: {} tests: run_inputs: From a52682aef8ee43cee5a9fed63fb00beae8d4a912 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Aug 2025 21:45:41 +0530 Subject: [PATCH 077/124] Support docker run for install-gcc --- script/install-gcc-src/meta.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/install-gcc-src/meta.yaml b/script/install-gcc-src/meta.yaml index 741ab39b6..edb79119b 100644 --- a/script/install-gcc-src/meta.yaml +++ b/script/install-gcc-src/meta.yaml @@ -37,6 +37,10 @@ tags: - gcc - src-gcc uid: faae0ebd6e1242db +docker: + skip_run_cmd: true + pre_run_cmds: + - mlc pull repo tests: run_inputs: - {} From 036b78169c0c4d09acd46a9ab7cc60db555eca83 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 3 Aug 2025 16:16:00 +0000 Subject: [PATCH 078/124] [Automated Commit] Format Codebase [skip ci] --- automation/script/docker.py | 1 - automation/script/docker_utils.py | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index dea7f3fb8..bfa639533 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -393,7 +393,6 @@ def docker_run(self_module, i): docker_inputs['mounts'] = res['mounts'] container_env_string = res['container_env_string'] - res = update_docker_environment( docker_settings, env, self_module.host_env_keys, container_env_string) if res['return'] > 0: diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index ca7ab2af6..6e2d430ac 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -40,9 +40,9 @@ def process_mounts(mounts, env, docker_settings, f_run_cmd, run_state): if 'mounts' in docker_settings: mounts.extend(docker_settings['mounts']) - for key in [ "MLC_INPUT", "MLC_OUTPUT", "MLC_OUTDIRNAME" ]: - if "${{ "+key+" }}:${{ "+key+" }}" not in mounts: - mounts.append("${{ "+key+" }}:${{ "+key+" }}") + for key in ["MLC_INPUT", "MLC_OUTPUT", "MLC_OUTDIRNAME"]: + if "${{ " + key + " }}:${{ " + key + " }}" not in mounts: + mounts.append("${{ " + key + " }}:${{ " + key + " }}") docker_input_mapping = docker_settings.get("input_mapping", {}) container_env_string = "" @@ -186,7 +186,8 @@ def prepare_docker_inputs(input_params, docker_settings, return docker_inputs, dockerfile_path -def update_docker_environment(docker_settings, env, host_env_keys, container_env_string): +def update_docker_environment( + docker_settings, env, host_env_keys, container_env_string): """ Updates the Docker environment variables and build arguments. From 3142abde64f122224cea9dbc525dcf52abd90eb7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Aug 2025 21:12:30 +0100 Subject: [PATCH 079/124] Update customize.py | Fix docker error on gh actions --- script/run-docker-container/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 62eb7f535..c319b1dfc 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -62,7 +62,8 @@ def preprocess(i): if os_info['platform'] == 'windows': CMD += " 2> nul" else: - CMD += " 2> /dev/null" + CMD += " 2> /dev/null || true " + logger.info(' ' + CMD) logger.info('') From 67e3a5d7b7d8db2267db01284bf16a10feef9f83 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 3 Aug 2025 20:13:03 +0000 Subject: [PATCH 080/124] [Automated Commit] Format Codebase [skip ci] --- script/run-docker-container/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index f084c59f1..2170172bd 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -63,7 +63,7 @@ def preprocess(i): CMD += " 2> nul" else: CMD += " 2> /dev/null || true " - + logger.info(' ' + CMD) logger.info('') From a477aac9d3d370a55d8bcefeb3bfae2c4d8cc817 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Aug 2025 21:17:55 +0100 Subject: [PATCH 081/124] Update customize.py | Fix docker error on gh actions --- script/run-docker-container/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 2170172bd..3c40bac93 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -101,7 +101,7 @@ def preprocess(i): if os_info['platform'] == 'windows': CMD += " 2> nul" else: - CMD += " 2> /dev/null" + CMD += " 2> /dev/null || true" logger.info('') logger.info('Checking Docker images:') From 876c15922ef9224f92a1988dd6cb6262cd23688a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Aug 2025 22:34:45 +0100 Subject: [PATCH 082/124] Fix permission error for fake runs --- automation/script/module.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 9c2a3a3ea..a9fa151db 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -1755,9 +1755,10 @@ def _run(self, i): env['MLC_TMP_CURRENT_PATH'], env['MLC_OUTDIRNAME']) env['MLC_OUTDIRNAME'] = c_outdirname - if not os.path.exists(c_outdirname): - os.makedirs(c_outdirname) - os.chdir(c_outdirname) + if not fake_run: #prevent permission error inside docker runs + if not os.path.exists(c_outdirname): + os.makedirs(c_outdirname) + os.chdir(c_outdirname) # Check if pre-process and detect if 'preprocess' in dir(customize_code) and not fake_run: From e1a3225ff5dcf603ef0a88d1bc78de85c8fe86f9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 3 Aug 2025 21:35:02 +0000 Subject: [PATCH 083/124] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index a9fa151db..ff2cd2455 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -1755,7 +1755,7 @@ def _run(self, i): env['MLC_TMP_CURRENT_PATH'], env['MLC_OUTDIRNAME']) env['MLC_OUTDIRNAME'] = c_outdirname - if not fake_run: #prevent permission error inside docker runs + if not fake_run: # prevent permission error inside docker runs if not os.path.exists(c_outdirname): os.makedirs(c_outdirname) os.chdir(c_outdirname) From 2e93356e2b7cb953dc3728469fd0798e444cb580 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 3 Aug 2025 23:03:41 +0100 Subject: [PATCH 084/124] Fix default input keys in docker run command --- automation/script/docker_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 6e2d430ac..7595fa274 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -339,6 +339,8 @@ def rebuild_flags( keys = sorted(command_dict.keys(), key=lambda x: x != "tags") for key in keys: + if key in [ "input", "output", "outdirname" ]: + continue # We have the corresponding env keys in container env string # Construct the full key with the prefix. full_key = f"{prefix}.{key}" if prefix else key From 89df712ede1378b8edfe85155ce8757b86c6b0ea Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 3 Aug 2025 22:04:01 +0000 Subject: [PATCH 085/124] [Automated Commit] Format Codebase [skip ci] --- automation/script/docker_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/script/docker_utils.py b/automation/script/docker_utils.py index 7595fa274..8fe73d47f 100644 --- a/automation/script/docker_utils.py +++ b/automation/script/docker_utils.py @@ -339,8 +339,8 @@ def rebuild_flags( keys = sorted(command_dict.keys(), key=lambda x: x != "tags") for key in keys: - if key in [ "input", "output", "outdirname" ]: - continue # We have the corresponding env keys in container env string + if key in ["input", "output", "outdirname"]: + continue # We have the corresponding env keys in container env string # Construct the full key with the prefix. full_key = f"{prefix}.{key}" if prefix else key From 9c15ba7dcfb5d952f2fffd01f97fa063f5625466 Mon Sep 17 00:00:00 2001 From: Thaw Zin Htoo <112410262+kamieyy@users.noreply.github.com> Date: Tue, 5 Aug 2025 11:06:40 +0700 Subject: [PATCH 086/124] Update meta.yaml removed - MLC_OUTDIRNAME names: - whisper-outdir-setup tags: setup,ml-model --- script/get-ml-model-whisper/meta.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index b4f128bcd..402b1908d 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -41,12 +41,6 @@ prehook_deps: names: - hf-zoo tags: get,ml-model,huggingface,zoo,_clone-repo -- force_env_keys: - - MLC_OUTDIRNAME - names: - - whisper-outdir-setup - tags: setup,ml-model - variations: fp32: From 417c23dac17937329c5fe3d2ea62d018ee30257f Mon Sep 17 00:00:00 2001 From: kamieyy Date: Tue, 5 Aug 2025 13:10:40 +0700 Subject: [PATCH 087/124] download models changed back to 'dae', cleaned prehook_deps and multiple defaults --- script/get-ml-model-whisper/meta.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index 402b1908d..3c1882575 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -41,7 +41,7 @@ prehook_deps: names: - hf-zoo tags: get,ml-model,huggingface,zoo,_clone-repo - + variations: fp32: default: true @@ -85,7 +85,6 @@ variations: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: - default: true env: MLC_DOWNLOAD_SRC: mlcommons group: download-src @@ -101,14 +100,14 @@ variations: force_env_keys: - MLC_OUTDIRNAME names: - - whisper-model-dae + - dae tags: download-and-extract update_tags_from_env_with_prefix: _url.: - MLC_DOWNLOAD_URL rclone: adr: - whisper-model-dae: + dae: tags: _rclone env: MLC_DOWNLOAD_URL: 'mlc-inference:mlcommons-inference-wg-public/Whisper/model/' @@ -128,7 +127,7 @@ variations: group: download-tool r2-downloader: adr: - whisper-model-dae: + dae: tags: _r2-downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' From 53a5b047b1a84436a61b304ee7b1f95e2b0e1aa7 Mon Sep 17 00:00:00 2001 From: Thaw Zin Htoo <112410262+kamieyy@users.noreply.github.com> Date: Wed, 6 Aug 2025 03:27:41 +0700 Subject: [PATCH 088/124] changed dependency name 'dae' to 'whisper-model-dae' for rclone and r2-downloader adr --- script/get-ml-model-whisper/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index 3c1882575..c8aef2f99 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -107,7 +107,7 @@ variations: - MLC_DOWNLOAD_URL rclone: adr: - dae: + whisper-model-dae: tags: _rclone env: MLC_DOWNLOAD_URL: 'mlc-inference:mlcommons-inference-wg-public/Whisper/model/' @@ -127,7 +127,7 @@ variations: group: download-tool r2-downloader: adr: - dae: + whisper-model-dae: tags: _r2-downloader env: MLC_DOWNLOAD_URL: 'https://inference.mlcommons-storage.org/metadata/whisper-model.uri' From 4b0e85bc32158a5f9c96992418c737376a92b04e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Aug 2025 03:31:37 +0530 Subject: [PATCH 089/124] Fix nvidia-mlperf-inference-v5.0-dev Fix nvidia-mlperf-inference-v5.0-dev --- automation/script/docker.py | 4 ++-- script/app-mlperf-inference/meta.yaml | 3 +-- script/get-ml-model-gptj/meta.yaml | 2 +- script/get-mlperf-inference-nvidia-common-code/meta.yaml | 4 ++++ script/get-mlperf-inference-results/meta.yaml | 6 ++++++ 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/automation/script/docker.py b/automation/script/docker.py index bfa639533..c97668ac3 100644 --- a/automation/script/docker.py +++ b/automation/script/docker.py @@ -103,7 +103,7 @@ def dockerfile(self_module, input_params): deps = docker_settings.get('build_deps', []) if deps: r = self_module._run_deps( - deps, [], env, {}, {}, {}, {}, '', [], '', False, '', + deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', show_time, ' ', run_state) if r['return'] > 0: return r @@ -336,7 +336,7 @@ def docker_run(self_module, i): deps = docker_settings.get('deps', []) if deps: r = self_module._run_deps( - deps, [], env, {}, {}, {}, {}, '', [], '', False, '', + deps, [], env, {}, {}, {}, add_deps_recursive, '', [], '', False, '', show_time, ' ', run_state) if r['return'] > 0: return r diff --git a/script/app-mlperf-inference/meta.yaml b/script/app-mlperf-inference/meta.yaml index 61b674cdb..f9ad1283f 100644 --- a/script/app-mlperf-inference/meta.yaml +++ b/script/app-mlperf-inference/meta.yaml @@ -2079,8 +2079,7 @@ variations: reproducibility add_deps_recursive: nvidia-inference-common-code: - version: r4.1-dev - tags: _mlcommons + tags: _mlcommons,_v4.0 nvidia-inference-server: version: r4.0 tags: _mlcommons diff --git a/script/get-ml-model-gptj/meta.yaml b/script/get-ml-model-gptj/meta.yaml index b54e3adeb..e20408f11 100644 --- a/script/get-ml-model-gptj/meta.yaml +++ b/script/get-ml-model-gptj/meta.yaml @@ -154,7 +154,6 @@ variations: - names: - cuda tags: get,cuda - - tags: get,nvidia,scratch,space - tags: get,cuda-devices,_with-pycuda enable_if_env: MLC_HOST_OS_FLAVOR: @@ -163,6 +162,7 @@ variations: skip_if_env: MLC_HOST_OS_FLAVOR: - ubuntu + - tags: get,nvidia,scratch,space - env: {} force_new_env_keys: - GPTJ_CHECKPOINT_PATH diff --git a/script/get-mlperf-inference-nvidia-common-code/meta.yaml b/script/get-mlperf-inference-nvidia-common-code/meta.yaml index e5d4a42dc..76fa3b666 100644 --- a/script/get-mlperf-inference-nvidia-common-code/meta.yaml +++ b/script/get-mlperf-inference-nvidia-common-code/meta.yaml @@ -41,6 +41,10 @@ variations: adr: mlperf-inference-results: tags: _nvidia-only + v4.0: + add_deps_recursive: + mlperf-inference-results: + tags: _code-only,_v4.0 v5.0: add_deps_recursive: mlperf-inference-results: diff --git a/script/get-mlperf-inference-results/meta.yaml b/script/get-mlperf-inference-results/meta.yaml index d3b886de3..f669e7477 100644 --- a/script/get-mlperf-inference-results/meta.yaml +++ b/script/get-mlperf-inference-results/meta.yaml @@ -69,6 +69,12 @@ variations: GITHUB_REPO_OWNER: GATEOverflow NVIDIA_ONLY: 'yes' group: source-repo + v4.0: + group: version + env: + MLC_GIT_URL: https://github.com/<<>>/inference_results_v4.0.git + MLC_MLPERF_INFERENCE_RESULTS_VERSION_NAME: v4.0 + MLC_VERSION: "v4.0" v5.0: group: version env: From 0de447d71598fe39d90c9a14cfef095776e69138 Mon Sep 17 00:00:00 2001 From: Thaw Zin Htoo <112410262+kamieyy@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:52:17 +0700 Subject: [PATCH 090/124] set mlc as default download-src --- script/get-ml-model-whisper/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index c8aef2f99..075f1defe 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -85,6 +85,7 @@ variations: env: MLC_DOWNLOAD_EXTRA_OPTIONS: -x mlc: + default: true env: MLC_DOWNLOAD_SRC: mlcommons group: download-src From 5647b645a6045fc83324398a576a2632eec2fc34 Mon Sep 17 00:00:00 2001 From: Thaw Zin Htoo <112410262+kamieyy@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:58:42 +0700 Subject: [PATCH 091/124] changed mlc dependency name to 'whisper-model-dae' --- script/get-ml-model-whisper/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index 075f1defe..7eabd926a 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -101,7 +101,7 @@ variations: force_env_keys: - MLC_OUTDIRNAME names: - - dae + - whisper-model-dae tags: download-and-extract update_tags_from_env_with_prefix: _url.: From 2f6e9d9f6fee112825b254b338989561f0f81485 Mon Sep 17 00:00:00 2001 From: Thaw Zin Htoo <112410262+kamieyy@users.noreply.github.com> Date: Wed, 6 Aug 2025 17:09:19 +0700 Subject: [PATCH 092/124] comment out - huggingface --- script/get-ml-model-whisper/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-whisper/meta.yaml b/script/get-ml-model-whisper/meta.yaml index 7eabd926a..c5cc34084 100644 --- a/script/get-ml-model-whisper/meta.yaml +++ b/script/get-ml-model-whisper/meta.yaml @@ -24,7 +24,7 @@ tests: - variations_list: - rclone,mlc,dry-run - r2-downloader,mlc,dry-run - - huggingface + # - huggingface uid: 3bea2356e97f47b1 From 3389752b2e3a4d72b97b2645b39b42cdb93be2d3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Aug 2025 12:59:37 +0000 Subject: [PATCH 093/124] [Automated Commit] Format Codebase [skip ci] --- script/get-ml-model-whisper/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-ml-model-whisper/customize.py b/script/get-ml-model-whisper/customize.py index a0f46ab85..0eb19e75e 100644 --- a/script/get-ml-model-whisper/customize.py +++ b/script/get-ml-model-whisper/customize.py @@ -20,13 +20,13 @@ def preprocess(i): # Normalize and expand the path checkpoint_path = os.path.abspath(os.path.expanduser(checkpoint_path)) env['MLC_ML_MODEL_WHISPER_PATH'] = checkpoint_path - + if not os.path.exists(checkpoint_path): return { 'return': 1, 'error': f"Provided Whisper model path '{checkpoint_path}' does not exist." } - + if not os.path.isdir(checkpoint_path): return { 'return': 1, From 8e988ed331aac9e465b936a5da2c97020d9ba250 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 6 Aug 2025 20:54:54 +0530 Subject: [PATCH 094/124] Added validate_cache option to handle softlinks for get-gcc --- script/get-gcc/validate_cache.bat | 23 +++++++++++++++++++++++ script/get-gcc/validate_cache.sh | 11 +++++++++++ 2 files changed, 34 insertions(+) create mode 100644 script/get-gcc/validate_cache.bat create mode 100644 script/get-gcc/validate_cache.sh diff --git a/script/get-gcc/validate_cache.bat b/script/get-gcc/validate_cache.bat new file mode 100644 index 000000000..dc4ceb197 --- /dev/null +++ b/script/get-gcc/validate_cache.bat @@ -0,0 +1,23 @@ +@echo off +setlocal enabledelayedexpansion + +:: Check if MLC_GCC_DIR_PATH is defined and exists +if not defined MLC_GCC_DIR_PATH ( + goto end +) +if not exist "%MLC_GCC_DIR_PATH%" ( + goto end +) + +:: Resolve absolute paths using PowerShell +for /f "delims=" %%A in ('powershell -NoProfile -Command "(Resolve-Path -LiteralPath \"%MLC_GCC_DIR_PATH%\").Path"') do set "resolvedPath1=%%A" +for /f "delims=" %%B in ('powershell -NoProfile -Command "(Resolve-Path -LiteralPath \"%MLC_GCC_INSTALLED_PATH%\").Path"') do set "resolvedPath2=%%B" + +:: Compare the resolved paths (case-insensitive on Windows) +if /i not "!resolvedPath1!"=="!resolvedPath2!" ( + exit /b 1 +) + +:end +exit /b 0 + diff --git a/script/get-gcc/validate_cache.sh b/script/get-gcc/validate_cache.sh new file mode 100644 index 000000000..28b0ee3fe --- /dev/null +++ b/script/get-gcc/validate_cache.sh @@ -0,0 +1,11 @@ +#!/bin/bash + + +if [[ -n "$MLC_GCC_DIR_PATH" && -e "$MLC_GCC_DIR_PATH" ]]; then + if [[ "$(realpath "$MLC_GCC_DIR_PATH")" != "$(realpath "$MLC_GCC_INSTALLED_PATH")" ]]; then + exit 1 + fi +fi + +test $? -eq 0 || exit $? +exit 0 From 3e5865272db0a91e57592659808410b591854a10 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 6 Aug 2025 22:08:02 +0530 Subject: [PATCH 095/124] Added validate_cache option to handle softlinks for get-gcc --- automation/script/module.py | 22 ++++++++++++++++++++-- script/get-gcc/customize.py | 7 +++---- script/get-gcc/run.sh | 2 +- script/get-gcc/validate_cache.sh | 4 ++-- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index ff2cd2455..77b14e3cd 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4871,10 +4871,28 @@ def find_cached_script(i): 'customize_code': customize_code, 'customize_common_input': customize_common_input } + env_tmp = copy.deepcopy(env) + path_to_cached_state_file = os.path.join(cached_script.path, + self_obj.file_with_cached_state) + + r = utils.load_json(file_name=path_to_cached_state_file) + if r['return'] > 0: + continue + + cached_meta = r.get("meta") + if not cached_meta: + continue + new_env = cached_meta.get("new_env", {}) + if new_env: + env_tmp.update(new_env) + state_tmp = copy.deepcopy(state) + new_state = cached_meta.get("new_state", {}) + if new_state: + state_tmp.update(new_state) deps = meta.get('deps') if deps: - r = self_obj._call_run_deps(deps, self_obj.local_env_keys, meta.get('local_env_keys', []), env, state, const, const_state, add_deps_recursive, + r = self_obj._call_run_deps(deps, self_obj.local_env_keys, meta.get('local_env_keys', []), env_tmp, state_tmp, const, const_state, add_deps_recursive, recursion_spaces + extra_recursion_spaces, remembered_selections, variation_tags_string, True, '', show_time, extra_recursion_spaces, {}) if r['return'] > 0: @@ -4882,7 +4900,7 @@ def find_cached_script(i): ii = { 'run_script_input': run_script_input, - 'env': env, + 'env': env_tmp, 'script_name': 'validate_cache', 'detect_version': True } diff --git a/script/get-gcc/customize.py b/script/get-gcc/customize.py index 9c21b8639..a47b321d3 100644 --- a/script/get-gcc/customize.py +++ b/script/get-gcc/customize.py @@ -83,11 +83,10 @@ def postprocess(i): found_file_path = env['MLC_GCC_BIN_WITH_PATH'] - found_path = os.path.dirname(found_file_path) - - env['MLC_GCC_INSTALLED_PATH'] = os.path.dirname( - found_path) # /usr in case of /usr/bin/gcc + found_path = os.path.dirname(os.path.realpath(found_file_path)) + env['MLC_GCC_INSTALLED_PATH'] = os.path.dirname(found_path) # /usr in case of /usr/bin/gcc + file_name_c = os.path.basename(found_file_path) # G: changed next line to handle cases like gcc-8 file_name_cpp = file_name_c.replace('gcc', 'g++') diff --git a/script/get-gcc/run.sh b/script/get-gcc/run.sh index e5b397bf6..f6311cbd4 100644 --- a/script/get-gcc/run.sh +++ b/script/get-gcc/run.sh @@ -3,6 +3,6 @@ gcc_bin=${MLC_GCC_BIN_WITH_PATH} echo "${gcc_bin} --version" ${gcc_bin} --version > tmp-ver.out -test $? -eq 0 || exit 1 +test $? -eq 0 || exit $? cat tmp-ver.out diff --git a/script/get-gcc/validate_cache.sh b/script/get-gcc/validate_cache.sh index 28b0ee3fe..9907116d2 100644 --- a/script/get-gcc/validate_cache.sh +++ b/script/get-gcc/validate_cache.sh @@ -1,11 +1,11 @@ #!/bin/bash - if [[ -n "$MLC_GCC_DIR_PATH" && -e "$MLC_GCC_DIR_PATH" ]]; then if [[ "$(realpath "$MLC_GCC_DIR_PATH")" != "$(realpath "$MLC_GCC_INSTALLED_PATH")" ]]; then + echo "$MLC_GCC_DIR_PATH" has changed from the cached entry. Invalidating the cache entry. exit 1 fi fi +${MLC_GCC_INSTALLED_PATH}/bin/gcc --version > tmp-ver.out test $? -eq 0 || exit $? -exit 0 From a2183dfde78f84d1739ffe70eddaef4c199734b7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Aug 2025 16:38:24 +0000 Subject: [PATCH 096/124] [Automated Commit] Format Codebase [skip ci] --- automation/script/module.py | 2 +- script/get-gcc/customize.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 77b14e3cd..827b7f8cc 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4873,7 +4873,7 @@ def find_cached_script(i): } env_tmp = copy.deepcopy(env) path_to_cached_state_file = os.path.join(cached_script.path, - self_obj.file_with_cached_state) + self_obj.file_with_cached_state) r = utils.load_json(file_name=path_to_cached_state_file) if r['return'] > 0: diff --git a/script/get-gcc/customize.py b/script/get-gcc/customize.py index a47b321d3..06bb2a137 100644 --- a/script/get-gcc/customize.py +++ b/script/get-gcc/customize.py @@ -85,8 +85,9 @@ def postprocess(i): found_path = os.path.dirname(os.path.realpath(found_file_path)) - env['MLC_GCC_INSTALLED_PATH'] = os.path.dirname(found_path) # /usr in case of /usr/bin/gcc - + env['MLC_GCC_INSTALLED_PATH'] = os.path.dirname( + found_path) # /usr in case of /usr/bin/gcc + file_name_c = os.path.basename(found_file_path) # G: changed next line to handle cases like gcc-8 file_name_cpp = file_name_c.replace('gcc', 'g++') From 4a50fadc05be89c799be684539bf96109e6030d3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Aug 2025 18:04:47 +0100 Subject: [PATCH 097/124] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 55e79c0ae..86b93ac06 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -5,7 +5,6 @@ on: types: [published] workflow_dispatch: {} - jobs: build_wheels: if: github.repository_owner == 'mlcommons' From 387d2808f761caf63b69639c9127bf4f92c1a550 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 6 Aug 2025 22:53:25 +0530 Subject: [PATCH 098/124] For now continue keeping gcc cache path as softlink --- script/get-gcc/customize.py | 3 ++- script/get-gcc/validate_cache.bat | 23 ------------------- ...alidate_cache.sh => validate_cache_off.sh} | 0 3 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 script/get-gcc/validate_cache.bat rename script/get-gcc/{validate_cache.sh => validate_cache_off.sh} (100%) diff --git a/script/get-gcc/customize.py b/script/get-gcc/customize.py index 06bb2a137..7e9f45a5e 100644 --- a/script/get-gcc/customize.py +++ b/script/get-gcc/customize.py @@ -83,7 +83,8 @@ def postprocess(i): found_file_path = env['MLC_GCC_BIN_WITH_PATH'] - found_path = os.path.dirname(os.path.realpath(found_file_path)) + #found_path = os.path.dirname(os.path.realpath(found_file_path)) //Need to use this if we don't want to cache specific to softlinks + found_path = os.path.dirname(os.path.abspath(found_file_path)) env['MLC_GCC_INSTALLED_PATH'] = os.path.dirname( found_path) # /usr in case of /usr/bin/gcc diff --git a/script/get-gcc/validate_cache.bat b/script/get-gcc/validate_cache.bat deleted file mode 100644 index dc4ceb197..000000000 --- a/script/get-gcc/validate_cache.bat +++ /dev/null @@ -1,23 +0,0 @@ -@echo off -setlocal enabledelayedexpansion - -:: Check if MLC_GCC_DIR_PATH is defined and exists -if not defined MLC_GCC_DIR_PATH ( - goto end -) -if not exist "%MLC_GCC_DIR_PATH%" ( - goto end -) - -:: Resolve absolute paths using PowerShell -for /f "delims=" %%A in ('powershell -NoProfile -Command "(Resolve-Path -LiteralPath \"%MLC_GCC_DIR_PATH%\").Path"') do set "resolvedPath1=%%A" -for /f "delims=" %%B in ('powershell -NoProfile -Command "(Resolve-Path -LiteralPath \"%MLC_GCC_INSTALLED_PATH%\").Path"') do set "resolvedPath2=%%B" - -:: Compare the resolved paths (case-insensitive on Windows) -if /i not "!resolvedPath1!"=="!resolvedPath2!" ( - exit /b 1 -) - -:end -exit /b 0 - diff --git a/script/get-gcc/validate_cache.sh b/script/get-gcc/validate_cache_off.sh similarity index 100% rename from script/get-gcc/validate_cache.sh rename to script/get-gcc/validate_cache_off.sh From f48ea9981f690c388e51fd163e65b1beb19dd430 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 Aug 2025 17:23:48 +0000 Subject: [PATCH 099/124] [Automated Commit] Format Codebase [skip ci] --- script/get-gcc/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-gcc/customize.py b/script/get-gcc/customize.py index 7e9f45a5e..9cc1c472a 100644 --- a/script/get-gcc/customize.py +++ b/script/get-gcc/customize.py @@ -83,7 +83,8 @@ def postprocess(i): found_file_path = env['MLC_GCC_BIN_WITH_PATH'] - #found_path = os.path.dirname(os.path.realpath(found_file_path)) //Need to use this if we don't want to cache specific to softlinks + # found_path = os.path.dirname(os.path.realpath(found_file_path)) //Need + # to use this if we don't want to cache specific to softlinks found_path = os.path.dirname(os.path.abspath(found_file_path)) env['MLC_GCC_INSTALLED_PATH'] = os.path.dirname( From 9d9c06f9ddfc041e2462a22f4d683d5fcafdadbc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 6 Aug 2025 18:28:38 +0100 Subject: [PATCH 100/124] Update build_wheels.yml --- .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 86b93ac06..55e79c0ae 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -5,6 +5,7 @@ on: types: [published] workflow_dispatch: {} + jobs: build_wheels: if: github.repository_owner == 'mlcommons' From 2c3b0d3d841b7b60380e27ef0c38bea07e9d1b87 Mon Sep 17 00:00:00 2001 From: hemanthtanguturi1 Date: Thu, 7 Aug 2025 17:04:48 +0530 Subject: [PATCH 101/124] Adding support to latest intel oneapi compiler Added Intel Oneapi 2025.2.0 version support --- script/get-oneapi/meta.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/script/get-oneapi/meta.yaml b/script/get-oneapi/meta.yaml index 69bcd435f..fca5be78e 100644 --- a/script/get-oneapi/meta.yaml +++ b/script/get-oneapi/meta.yaml @@ -4,7 +4,7 @@ automation_uid: 5b4e0237da074764 cache: true category: Compiler automation clean_files: [] -default_version: 2025.1.1 +default_version: 2025.2.0 deps: - tags: detect,os name: Detect or install OneAPI compiler @@ -35,12 +35,12 @@ tests: - fortran versions: - 2025.1.1: + 2025.2.0: env: - MLC_ONEAPI_INSTALL_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/6bfca885-4156-491e-849b-1cd7da9cc760 - MLC_ONEAPI_INSTALL_FILENAME: intel-oneapi-base-toolkit-2025.1.1.36_offline.sh - MLC_ONEAPI_INSTALL_VERSION_PREFIX: '2025.1' - MLC_VERSION: '2025.1.1' + MLC_ONEAPI_INSTALL_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/bd1d0273-a931-4f7e-ab76-6a2a67d646c7 + MLC_ONEAPI_INSTALL_FILENAME: intel-oneapi-base-toolkit-2025.2.0.592.sh + MLC_ONEAPI_INSTALL_VERSION_PREFIX: '2025.2' + MLC_VERSION: '2025.2.0' variations: path.#: From 13ad1cc511a9e396f82d0b4ef26ed0fd63778b03 Mon Sep 17 00:00:00 2001 From: hemanthtanguturi1 Date: Thu, 7 Aug 2025 19:15:06 +0530 Subject: [PATCH 102/124] Update meta.yaml --- script/get-oneapi/meta.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/script/get-oneapi/meta.yaml b/script/get-oneapi/meta.yaml index fca5be78e..5f9ad5b1f 100644 --- a/script/get-oneapi/meta.yaml +++ b/script/get-oneapi/meta.yaml @@ -42,6 +42,13 @@ versions: MLC_ONEAPI_INSTALL_VERSION_PREFIX: '2025.2' MLC_VERSION: '2025.2.0' + 2025.1.0: + env: + MLC_ONEAPI_INSTALL_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/6bfca885-4156-491e-849b-1cd7da9cc760 + MLC_ONEAPI_INSTALL_FILENAME: intel-oneapi-base-toolkit-2025.1.1.36_offline.sh + MLC_ONEAPI_INSTALL_VERSION_PREFIX: '2025.1' + MLC_VERSION: '2025.1.1' + variations: path.#: env: From f01a9d758701559685d817e7b8704b39511b0b61 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Aug 2025 01:21:30 +0530 Subject: [PATCH 103/124] export src repo hash for install llvm and gcc --- script/install-gcc-src/customize.py | 8 ++++++++ script/install-gcc-src/meta.yaml | 3 +++ script/install-llvm-src/customize.py | 3 +++ 3 files changed, 14 insertions(+) diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index 7edf5103b..d0e4671db 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -20,3 +20,11 @@ def preprocess(i): env['MLC_GCC_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') return {'return': 0} + +def postprocess(i): + + env = i['env'] + if env.get('MLC_GIT_REPO_CURRENT_HASH', '') != '': + env['MLC_GCC_SRC_REPO_COMMIT'] = env['MLC_GIT_REPO_CURRENT_HASH'] + + return {'return': 0} diff --git a/script/install-gcc-src/meta.yaml b/script/install-gcc-src/meta.yaml index edb79119b..9872c0d35 100644 --- a/script/install-gcc-src/meta.yaml +++ b/script/install-gcc-src/meta.yaml @@ -26,6 +26,9 @@ deps: - MLC_GIT_CHECKOUT_TAG env: MLC_GIT_URL: git://gcc.gnu.org/git/gcc.git + +new_env_keys: + - MLC_GCC_* post_deps: - skip_if_env: MLC_REQUIRE_INSTALL: diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index 2f0c25412..af3ba789a 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -83,4 +83,7 @@ def postprocess(i): if os.path.isdir(path_include): env['+C_INCLUDE_PATH'] = [path_include] + if env.get('MLC_GIT_REPO_CURRENT_HASH', '') != '': + env['MLC_LLVM_SRC_REPO_COMMIT'] = env['MLC_GIT_REPO_CURRENT_HASH'] + return {'return': 0} From e037d81a6e1f05f8cbe262deec68aa1872480968 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 7 Aug 2025 19:51:48 +0000 Subject: [PATCH 104/124] [Automated Commit] Format Codebase [skip ci] --- script/install-gcc-src/customize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index d0e4671db..39f97ba9d 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -21,10 +21,11 @@ def preprocess(i): return {'return': 0} + def postprocess(i): - + env = i['env'] if env.get('MLC_GIT_REPO_CURRENT_HASH', '') != '': env['MLC_GCC_SRC_REPO_COMMIT'] = env['MLC_GIT_REPO_CURRENT_HASH'] - + return {'return': 0} From 9966c1a09e7886f49c4b49c788ee5c6d96bdaff9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 8 Aug 2025 01:51:36 +0530 Subject: [PATCH 105/124] Support repo_url in doc script --- automation/script/doc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automation/script/doc.py b/automation/script/doc.py index 340d3805c..620442dc4 100644 --- a/automation/script/doc.py +++ b/automation/script/doc.py @@ -76,6 +76,8 @@ def get_setup_readme(script_repo): if '@' in repo_name: repo_name = repo_name.split('@')[1] + repo_location = script_repo.meta.get('url', repo_alias) + setup_readme = f"""`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` mkdir /mnt/$USER/MLC @@ -100,7 +102,7 @@ def get_setup_readme(script_repo): Once `mlcflow` is installed: ```bash -mlc pull repo {repo_alias} --pat= +mlc pull repo {repo_location} --pat= ``` - `--pat` or `--ssh` is only needed if the repo is PRIVATE - If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token From 5bb944fb16d51361640705a8af0e80ca96839924 Mon Sep 17 00:00:00 2001 From: amd-arsuresh Date: Thu, 14 Aug 2025 14:08:30 +0100 Subject: [PATCH 106/124] Support target for gcc,llvm build from src (#573) * Update the base docker image for mlperf automotive * Support target for gcc,llvm build from src --- script/app-mlperf-automotive/meta.yaml | 2 +- script/install-gcc-src/customize.py | 5 +++++ script/install-gcc-src/meta.yaml | 3 +++ script/install-gcc-src/run.sh | 5 ++++- script/install-llvm-src/customize.py | 2 +- script/install-llvm-src/meta.yaml | 3 +++ 6 files changed, 17 insertions(+), 3 deletions(-) diff --git a/script/app-mlperf-automotive/meta.yaml b/script/app-mlperf-automotive/meta.yaml index 3afa3a69e..d5a1bfedc 100644 --- a/script/app-mlperf-automotive/meta.yaml +++ b/script/app-mlperf-automotive/meta.yaml @@ -396,7 +396,7 @@ variations: tags: _cuda docker: all_gpus: 'yes' - base_image: nvcr.io/nvidia/pytorch:24.03-py3 + base_image: nvcr.io/nvidia/pytorch:24.08-py3 os_version: 22.04 v0.5: {} diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index 39f97ba9d..839a43699 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -17,6 +17,11 @@ def preprocess(i): recursion_spaces = i['recursion_spaces'] + if env.get('MLC_GCC_TARGET', '') != '': + env['MLC_GCC_TARGET_STRING'] = f""" --target={env['MLC_GCC_TARGET']} """ + else: + env['MLC_GCC_TARGET_STRING'] = '' + env['MLC_GCC_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') return {'return': 0} diff --git a/script/install-gcc-src/meta.yaml b/script/install-gcc-src/meta.yaml index 9872c0d35..74a883f26 100644 --- a/script/install-gcc-src/meta.yaml +++ b/script/install-gcc-src/meta.yaml @@ -24,6 +24,9 @@ deps: - MLC_GIT_CHECKOUT_SHA _tag.: - MLC_GIT_CHECKOUT_TAG +input_mapping: + targets: MLC_GCC_TARGET + env: MLC_GIT_URL: git://gcc.gnu.org/git/gcc.git diff --git a/script/install-gcc-src/run.sh b/script/install-gcc-src/run.sh index c8b1ca291..7d3131950 100644 --- a/script/install-gcc-src/run.sh +++ b/script/install-gcc-src/run.sh @@ -22,7 +22,10 @@ cd src ./contrib/download_prerequisites cd ../build -../src/configure --prefix="${INSTALL_DIR}" --with-gcc-major-version-only --disable-multilib + +cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} --with-gcc-major-version-only --disable-multilib" +echo $cmd +eval $cmd test $? -eq 0 || exit $? diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index af3ba789a..ae34b879e 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -52,7 +52,7 @@ def preprocess(i): targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86') - cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type } -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}""" + cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type} -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}""" env['MLC_LLVM_CMAKE_CMD'] = cmake_cmd diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index ca44d5003..9a5a9d5b0 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -36,6 +36,9 @@ deps: - MLC_GIT_CHECKOUT_TAG _tag.llvmorg-: - MLC_VERSION +input_mapping: + targets: MLC_LLVM_TARGETS_TO_BUILD + env: MLC_GIT_URL: https://github.com/llvm/llvm-project name: Build LLVM compiler from sources (can take >30 min) From 858a209cfbede99dacb72f870ab378895152b178 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 14 Aug 2025 19:03:16 +0530 Subject: [PATCH 107/124] Added --host option for install-gcc-src --- script/install-gcc-src/customize.py | 5 +++++ script/install-gcc-src/meta.yaml | 1 + script/install-gcc-src/run.sh | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index 839a43699..c13141d22 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -22,6 +22,11 @@ def preprocess(i): else: env['MLC_GCC_TARGET_STRING'] = '' + if env.get('MLC_GCC_HOST', '') != '': + env['MLC_GCC_HOST_STRING'] = f""" --target={env['MLC_GCC_HOST']} """ + else: + env['MLC_GCC_HOST_STRING'] = '' + env['MLC_GCC_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') return {'return': 0} diff --git a/script/install-gcc-src/meta.yaml b/script/install-gcc-src/meta.yaml index 74a883f26..3bc1e7083 100644 --- a/script/install-gcc-src/meta.yaml +++ b/script/install-gcc-src/meta.yaml @@ -26,6 +26,7 @@ deps: - MLC_GIT_CHECKOUT_TAG input_mapping: targets: MLC_GCC_TARGET + host: MLC_GCC_HOST env: MLC_GIT_URL: git://gcc.gnu.org/git/gcc.git diff --git a/script/install-gcc-src/run.sh b/script/install-gcc-src/run.sh index 7d3131950..d11bedbd0 100644 --- a/script/install-gcc-src/run.sh +++ b/script/install-gcc-src/run.sh @@ -23,7 +23,7 @@ cd src cd ../build -cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} --with-gcc-major-version-only --disable-multilib" +cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} ${MLC_GCC_HOST_STRING} --with-gcc-major-version-only --disable-multilib" echo $cmd eval $cmd From eb48f940ca54bd8adcc9f5198aacf569ef217d5f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 14 Aug 2025 14:33:48 +0100 Subject: [PATCH 108/124] Added --host option for install-gcc-src (#574) Co-authored-by: github-actions[bot] --- script/install-gcc-src/customize.py | 5 +++++ script/install-gcc-src/meta.yaml | 1 + script/install-gcc-src/run.sh | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index 839a43699..c13141d22 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -22,6 +22,11 @@ def preprocess(i): else: env['MLC_GCC_TARGET_STRING'] = '' + if env.get('MLC_GCC_HOST', '') != '': + env['MLC_GCC_HOST_STRING'] = f""" --target={env['MLC_GCC_HOST']} """ + else: + env['MLC_GCC_HOST_STRING'] = '' + env['MLC_GCC_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') return {'return': 0} diff --git a/script/install-gcc-src/meta.yaml b/script/install-gcc-src/meta.yaml index 74a883f26..3bc1e7083 100644 --- a/script/install-gcc-src/meta.yaml +++ b/script/install-gcc-src/meta.yaml @@ -26,6 +26,7 @@ deps: - MLC_GIT_CHECKOUT_TAG input_mapping: targets: MLC_GCC_TARGET + host: MLC_GCC_HOST env: MLC_GIT_URL: git://gcc.gnu.org/git/gcc.git diff --git a/script/install-gcc-src/run.sh b/script/install-gcc-src/run.sh index 7d3131950..d11bedbd0 100644 --- a/script/install-gcc-src/run.sh +++ b/script/install-gcc-src/run.sh @@ -23,7 +23,7 @@ cd src cd ../build -cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} --with-gcc-major-version-only --disable-multilib" +cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} ${MLC_GCC_HOST_STRING} --with-gcc-major-version-only --disable-multilib" echo $cmd eval $cmd From 556d6028af831df8e9470f253d887a56cb9d8a03 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 14 Aug 2025 23:45:16 +0530 Subject: [PATCH 109/124] Fix cross compilation of llvm --- script/get-generic-sys-util/meta.yaml | 7 +++++++ script/install-llvm-src/customize.py | 3 ++- script/install-llvm-src/meta.yaml | 5 +++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/script/get-generic-sys-util/meta.yaml b/script/get-generic-sys-util/meta.yaml index afc39aaec..74d023872 100644 --- a/script/get-generic-sys-util/meta.yaml +++ b/script/get-generic-sys-util/meta.yaml @@ -628,6 +628,13 @@ variations: brew: '' dnf: ntpdate yum: ntpdate + crossbuild-essential-arm64: + env: + MLC_SYS_UTIL_NAME: crossbuild-essential-arm64 + MLC_SYS_UTIL_CHECK_CMD: 'command -v aarch64-linux-gnu-gcc >/dev/null 2>&1' + state: + crossbuild-essential-arm64: + apt: crossbuild-essential-arm64 numactl: deps: - enable_if_env: diff --git a/script/install-llvm-src/customize.py b/script/install-llvm-src/customize.py index ae34b879e..30b6f67f7 100644 --- a/script/install-llvm-src/customize.py +++ b/script/install-llvm-src/customize.py @@ -51,8 +51,9 @@ def preprocess(i): llvm_build_type = env['MLC_LLVM_BUILD_TYPE'] targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86') + cross_compile_options = env.get('MLC_LLVM_CROSS_COMPILE_FLAGS', '') - cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type} -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}""" + cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type} -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {cross_compile_options} {extra_cmake_options}""" env['MLC_LLVM_CMAKE_CMD'] = cmake_cmd diff --git a/script/install-llvm-src/meta.yaml b/script/install-llvm-src/meta.yaml index 9a5a9d5b0..cef05d5a7 100644 --- a/script/install-llvm-src/meta.yaml +++ b/script/install-llvm-src/meta.yaml @@ -110,6 +110,11 @@ variations: env: MLC_LLVM_INSTALLED_PATH: '#' MLC_LLVM_USE_INSTALLED_DIR: yes + cross-compile-x86-aarch64: + deps: + - tags: get,generic-sys-util,_crossbuild-essential-arm64 + env: + MLC_LLVM_CROSS_COMPILE_FLAGS: "-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_PROCESSOR=aarch64 -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-linux-gnu -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++" for-intel-mlperf-inference-v3.1-bert: adr: conda-package: From 059270c156dd5622cfed289d9b3553d91e2ee592 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 15 Aug 2025 01:17:40 +0530 Subject: [PATCH 110/124] Support env in os_info --- automation/utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/automation/utils.py b/automation/utils.py index 98a3a3718..66aefc92b 100644 --- a/automation/utils.py +++ b/automation/utils.py @@ -14,12 +14,12 @@ def get_host_os_info(i={}): Get some host platform name (currently windows or linux) and OS bits Args: - (CM input dict): + (input dict): (bits) (str): force host platform bits Returns: - (CM return dict): + (dict): * return (int): return code == 0 if no error and >0 if error * (error) (str): error string if return>0 @@ -56,9 +56,9 @@ def get_host_os_info(i={}): } else: if platform.system().lower().startswith('darwin'): - platform = 'darwin' + platform_name = 'darwin' else: - platform = 'linux' + platform_name = 'linux' info['bat_ext'] = '.sh' info['set_env'] = 'export ${key}="${value}"' @@ -72,12 +72,14 @@ def get_host_os_info(i={}): info['start_script'] = ['#!/bin/bash', ''] info['env'] = {} - info['platform'] = platform + info['platform'] = platform_name + info['env']['MLC_HOST_PLATFORM_FLAVOR'] = platform.machine() + info['env']['MLC_HOST_OS_TYPE'] = platform_name obits = i.get('bits', '') if obits == '': obits = '32' - if platform == 'windows': + if platform_name == 'windows': # Trying to get fast way to detect bits if os.environ.get('ProgramW6432', '') != '' or os.environ.get( 'ProgramFiles(x86)', '') != '': # pragma: no cover From 8c0b5790e75d25573ba0c58da653ca554a735b01 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 15 Aug 2025 01:20:24 +0530 Subject: [PATCH 111/124] Support env in os_info --- automation/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/utils.py b/automation/utils.py index 66aefc92b..5881c148b 100644 --- a/automation/utils.py +++ b/automation/utils.py @@ -41,7 +41,7 @@ def get_host_os_info(i={}): pbits = str(8 * struct.calcsize("P")) if platform.system().lower().startswith('win'): - platform = 'windows' + platform_name = 'windows' info['bat_ext'] = '.bat' info['set_env'] = 'set ${key}=${value}' info['env_separator'] = ';' From 92fc89f2c6a54a7710d05ddd5624f2b8d372d849 Mon Sep 17 00:00:00 2001 From: nathanw-mlc Date: Fri, 15 Aug 2025 11:46:49 -0500 Subject: [PATCH 112/124] Update load-secrets-action to v3 with Windows runner compatability --- ...un_tests_on_modified_meta_with_secrets.yml | 27 ++++------ .../workflows/test-mlc-script-features.yml | 10 ++-- .github/workflows/test-mlperf-automotive.yml | 27 +++------- ...bert-deepsparse-tf-onnxruntime-pytorch.yml | 49 +++--------------- ...lperf-inference-mlcommons-cpp-resnet50.yml | 51 +++---------------- ...erf-inference-resnet50-closed-division.yml | 48 +++-------------- .../test-mlperf-inference-resnet50.yml | 48 +++-------------- .../test-mlperf-inference-tvm-resnet50.yml | 10 ++-- 8 files changed, 50 insertions(+), 220 deletions(-) diff --git a/.github/workflows/run_tests_on_modified_meta_with_secrets.yml b/.github/workflows/run_tests_on_modified_meta_with_secrets.yml index 3e2484ce3..8c8f84506 100644 --- a/.github/workflows/run_tests_on_modified_meta_with_secrets.yml +++ b/.github/workflows/run_tests_on_modified_meta_with_secrets.yml @@ -7,20 +7,6 @@ on: - 'script/**meta.yaml' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - gdrive_secret: ${{ steps.op-load-secret.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }} - steps: - - name: Load secret from 1Password - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - GDRIVE_SERVICE_ACCOUNT_KEY: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - get_modified_files: runs-on: ubuntu-latest outputs: @@ -52,7 +38,6 @@ jobs: process_modified_files: needs: - get_modified_files - - fetch-secret runs-on: ubuntu-latest if: needs.get_modified_files.outputs.processed_files != '[]' && needs.get_modified_files.outputs.processed_files != '' strategy: @@ -60,12 +45,18 @@ jobs: matrix: file_info: ${{ fromJSON(needs.get_modified_files.outputs.processed_files).file_info }} steps: + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + GDRIVE_SERVICE_ACCOUNT_KEY: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential + - name: Set RCLONE Service account env var from secret shell: bash run: | - echo "::add-mask::${{ needs.fetch-secret.outputs.gdrive_secret }}" - echo "RCLONE_CONFIG_MLC_COGNATA_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV - echo "RCLONE_CONFIG_MLC_NUSCENES_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV + echo "RCLONE_CONFIG_MLC_COGNATA_SERVICE_ACCOUNT_CREDENTIALS=${{ steps.op-load-secrets.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }}" >> $GITHUB_ENV + echo "RCLONE_CONFIG_MLC_NUSCENES_SERVICE_ACCOUNT_CREDENTIALS=${{ steps.op-load-secrets.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }}" >> $GITHUB_ENV - name: Process meta.yaml file run: | diff --git a/.github/workflows/test-mlc-script-features.yml b/.github/workflows/test-mlc-script-features.yml index 4570d5bdf..cbc532bb2 100644 --- a/.github/workflows/test-mlc-script-features.yml +++ b/.github/workflows/test-mlc-script-features.yml @@ -173,18 +173,16 @@ jobs: Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append } - - name: Load secret + - name: Load secrets if: github.repository_owner == 'mlcommons' && env.run_step == 'true' - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Push Results env: - GITHUB_TOKEN: ${{ steps.op-load-secret.outputs.PAT }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} if: github.repository_owner == 'mlcommons' && env.run_step == 'true' run: | git config --global user.name "mlcommons-bot" diff --git a/.github/workflows/test-mlperf-automotive.yml b/.github/workflows/test-mlperf-automotive.yml index a03560dc1..54c50a95e 100644 --- a/.github/workflows/test-mlperf-automotive.yml +++ b/.github/workflows/test-mlperf-automotive.yml @@ -7,35 +7,24 @@ on: - 'script/**meta.yaml' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - gdrive_secret: ${{ steps.op-load-secret.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }} - steps: - - name: Load secret from 1Password - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - GDRIVE_SERVICE_ACCOUNT_KEY: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - run-mlperf: runs-on: ubuntu-latest - needs: - - fetch-secret steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 2 + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + GDRIVE_SERVICE_ACCOUNT_KEY: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Set RCLONE Service account env var from secret shell: bash run: | - echo "::add-mask::${{ needs.fetch-secret.outputs.gdrive_secret }}" - echo "RCLONE_CONFIG_MLC_COGNATA_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV - echo "RCLONE_CONFIG_MLC_NUSCENES_SERVICE_ACCOUNT_CREDENTIALS=${{ needs.fetch-secret.outputs.gdrive_secret }}" >> $GITHUB_ENV + echo "RCLONE_CONFIG_MLC_COGNATA_SERVICE_ACCOUNT_CREDENTIALS=${{ steps.op-load-secrets.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }}" >> $GITHUB_ENV + echo "RCLONE_CONFIG_MLC_NUSCENES_SERVICE_ACCOUNT_CREDENTIALS=${{ steps.op-load-secrets.outputs.GDRIVE_SERVICE_ACCOUNT_KEY }}" >> $GITHUB_ENV - name: Run MLPerf run: | pip install mlcflow diff --git a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml index f4ded9274..e5925e99a 100644 --- a/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml +++ b/.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml @@ -9,35 +9,8 @@ on: - '!**.md' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT - build: name: MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }} - needs: [fetch-secret] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -85,25 +58,15 @@ jobs: else echo "run_step=false" >> $GITHUB_ENV fi - - name: Decrypt secret - id: decrypt-secret - shell: bash + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }} - run: | - echo "Running on OS: ${{ matrix.os }}" - - # Decrypt - decrypted=$(echo "$encrypted_secret" | \ - openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "::add-mask::$decrypted" - echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Push Results env: - GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} if: github.repository_owner == 'mlcommons' && env.run_step == 'true' run: | git config --global user.name "mlcommons-bot" diff --git a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml index 147fb0ff4..29acfbf1d 100644 --- a/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml @@ -8,36 +8,9 @@ on: - '**' - '!**.md' -jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT - +jobs: build: name: MLPerf inference MLCommons C++ ResNet50 - needs: [fetch-secret] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -80,25 +53,15 @@ jobs: else echo "run_step=false" >> $GITHUB_ENV fi - - name: Decrypt secret - id: decrypt-secret - shell: bash + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }} - run: | - echo "Running on OS: ${{ matrix.os }}" - - # Decrypt - decrypted=$(echo "$encrypted_secret" | \ - openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "::add-mask::$decrypted" - echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Push Results env: - GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} USER: mlcommons-bot EMAIL: mlcommons-bot@users.noreply.github.com if: github.repository_owner == 'mlcommons' && env.run_step == 'true' diff --git a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml index 663dff628..868ae873d 100644 --- a/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml +++ b/.github/workflows/test-mlperf-inference-resnet50-closed-division.yml @@ -5,35 +5,8 @@ on: - cron: '0 0 * * 0' # Runs once a week on Sunday at 00:00 UTC workflow_dispatch: {} # Allows manual triggering of the workflow jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT - build: name: MLPerf inference MLCommons ResNet50 Closed Division - needs: [fetch-secret] runs-on: ${{ matrix.os }} env: MLC_INDEX: "on" @@ -115,25 +88,16 @@ jobs: Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append } - - name: Decrypt secret - id: decrypt-secret - shell: bash + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }} - run: | - echo "Running on OS: ${{ matrix.os }}" + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - # Decrypt - decrypted=$(echo "$encrypted_secret" | \ - openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "::add-mask::$decrypted" - echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT - name: Push Results env: - GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} if: github.repository_owner == 'mlcommons' && env.run_step == 'true' run: | git config --global user.name "mlcommons-bot" diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index a99acc106..708d27b32 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -9,34 +9,7 @@ on: - '!**.md' jobs: - fetch-secret: - runs-on: ubuntu-latest - outputs: - encrypted_secret: ${{ steps.encrypt-secret.outputs.encrypted_secret }} - steps: - - name: Load secret - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false - env: - OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - - - name: Encrypt secret - id: encrypt-secret - env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - run: | - # AES-256 encrypt - encrypted=$(echo "${{ steps.op-load-secret.outputs.pat }}" | \ - openssl enc -e -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "encrypted_secret=$encrypted" >> $GITHUB_OUTPUT - mlc-run-with-results-upload: - needs: [fetch-secret] runs-on: ${{ matrix.os }} env: MLC_INDEX: "on" @@ -109,25 +82,16 @@ jobs: Write-Host "run_step=false" | Out-File -FilePath $Env:GITHUB_ENV -Append } - - name: Decrypt secret - id: decrypt-secret - shell: bash + - name: Load secrets + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: - ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} - encrypted_secret: ${{ needs.fetch-secret.outputs.encrypted_secret }} - run: | - echo "Running on OS: ${{ matrix.os }}" + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - # Decrypt - decrypted=$(echo "$encrypted_secret" | \ - openssl enc -d -aes-256-cbc -md sha512 -pbkdf2 -iter 100000 \ - -pass pass:"$ENCRYPTION_KEY" -base64 -A) - - echo "::add-mask::$decrypted" - echo "DECRYPTED_SECRET=$decrypted" >> $GITHUB_OUTPUT - name: Push Results env: - GITHUB_TOKEN: ${{ steps.decrypt-secret.outputs.decrypted_secret }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} if: github.repository_owner == 'mlcommons' && env.run_step == 'true' run: | git config --global user.name "mlcommons-bot" diff --git a/.github/workflows/test-mlperf-inference-tvm-resnet50.yml b/.github/workflows/test-mlperf-inference-tvm-resnet50.yml index 393d1c5cd..05411562e 100644 --- a/.github/workflows/test-mlperf-inference-tvm-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-tvm-resnet50.yml @@ -46,18 +46,16 @@ jobs: else echo "run_step=false" >> $GITHUB_ENV fi - - name: Load secret + - name: Load secrets if: github.repository_owner == 'mlcommons' && env.run_step == 'true' - id: op-load-secret - uses: 1password/load-secrets-action@v2 - with: - export-env: false + id: op-load-secrets + uses: 1password/load-secrets-action@v3 env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} PAT: op://7basd2jirojjckncf6qnq3azai/bzbaco3uxoqs2rcyu42rvuccga/credential - name: Push Results env: - GITHUB_TOKEN: ${{ steps.op-load-secret.outputs.PAT }} + GITHUB_TOKEN: ${{ steps.op-load-secrets.outputs.PAT }} USER: mlcommons-bot EMAIL: mlcommons-bot@users.noreply.github.com if: github.repository_owner == 'mlcommons' && env.run_step == 'true' From a015e7d2ca666aab4b1615edeb81e72852b0c803 Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 16 Aug 2025 08:50:22 +0530 Subject: [PATCH 113/124] Support more configs for jemalloc --- script/get-lib-jemalloc/customize.py | 10 +++++++++- script/get-lib-jemalloc/meta.yaml | 9 +++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/script/get-lib-jemalloc/customize.py b/script/get-lib-jemalloc/customize.py index 0fb3b7de8..e5a8439f4 100644 --- a/script/get-lib-jemalloc/customize.py +++ b/script/get-lib-jemalloc/customize.py @@ -1,8 +1,8 @@ +from utils import * from mlc import utils import os import subprocess - def preprocess(i): env = i['env'] @@ -15,6 +15,14 @@ def preprocess(i): configure_command += f""" --with-lg-quantum={env['MLC_JEMALLOC_LG_QUANTUM']} """ if env.get('MLC_JEMALLOC_LG_PAGE', '') != '': configure_command += f""" --with-lg-page={env['MLC_JEMALLOC_LG_PAGE']} """ + + if is_true(env.get('MLC_JEMALLOC_STATS')): + configure_command += " --enable-stats " + + if is_true(env.get('MLC_JEMALLOC_PROF')): + configure_command += " --enable-prof " + + if env.get('MLC_JEMALLOC_CONFIG', '') != '': configure_command += f""" {env['MLC_JEMALLOC_CONFIG'].replace("'", "")} """ diff --git a/script/get-lib-jemalloc/meta.yaml b/script/get-lib-jemalloc/meta.yaml index a856e9e4b..10c73be91 100644 --- a/script/get-lib-jemalloc/meta.yaml +++ b/script/get-lib-jemalloc/meta.yaml @@ -65,9 +65,18 @@ variations: config.#: env: MLC_JEMALLOC_CONFIG: '#' + enable-stats: + env: + MLC_JEMALLOC_STATS: true + enable-prof: + env: + MLC_JEMALLOC_PROF: true lg-page.#: env: MLC_JEMALLOC_LG_PAGE: '#' + lg-hugepage.#: + env: + MLC_JEMALLOC_LG_PAGE: '#' lg-quantum.#: env: MLC_JEMALLOC_LG_QUANTUM: '#' From 0eb254e73fc320c890e240157acc1058d33d2096 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 16 Aug 2025 03:20:38 +0000 Subject: [PATCH 114/124] [Automated Commit] Format Codebase [skip ci] --- script/get-lib-jemalloc/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/get-lib-jemalloc/customize.py b/script/get-lib-jemalloc/customize.py index e5a8439f4..4771bc813 100644 --- a/script/get-lib-jemalloc/customize.py +++ b/script/get-lib-jemalloc/customize.py @@ -3,6 +3,7 @@ import os import subprocess + def preprocess(i): env = i['env'] @@ -18,11 +19,10 @@ def preprocess(i): if is_true(env.get('MLC_JEMALLOC_STATS')): configure_command += " --enable-stats " - + if is_true(env.get('MLC_JEMALLOC_PROF')): configure_command += " --enable-prof " - if env.get('MLC_JEMALLOC_CONFIG', '') != '': configure_command += f""" {env['MLC_JEMALLOC_CONFIG'].replace("'", "")} """ From d03e129d24209401eb017046648920e85b38980c Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 17 Aug 2025 22:16:09 +0530 Subject: [PATCH 115/124] Fixes for help, support --all in search, update some outdated READMEs --- automation/script/help.py | 29 ++- automation/script/module.py | 2 + .../README.md | 10 +- script/app-mlperf-inference-nvidia/README.md | 212 +++++++++++++++++- script/app-mlperf-inference/README.md | 10 +- .../README.md | 6 +- script/download-file/README.md | 8 +- .../README.md | 6 +- script/get-cache-dir/README.md | 6 +- script/get-generic-sys-util/README.md | 7 +- script/get-git-repo/README.md | 6 +- script/get-ipol-src/meta.yaml | 6 +- script/get-lib-jemalloc/README.md | 49 +++- .../get-ml-model-abtf-ssd-pytorch/README.md | 6 +- script/get-ml-model-deeplabv3_plus/README.md | 6 +- script/get-ml-model-resnet50/README.md | 6 +- script/get-oneapi/README.md | 43 +++- script/get-spec-ptd/meta.yaml | 3 +- script/get-tensorrt/meta.yaml | 7 +- .../README.md | 10 +- script/submit-mlperf-results/README.md | 6 +- 21 files changed, 379 insertions(+), 65 deletions(-) diff --git a/automation/script/help.py b/automation/script/help.py index 9c2212931..f4d9ab3e5 100644 --- a/automation/script/help.py +++ b/automation/script/help.py @@ -32,20 +32,25 @@ def display_help(self_module, input_params): scripts_list = search_result['list'] if not scripts_list: - return {'return': 1, 'error': 'No scripts were found'} - generic_inputs = self_module.input_flags_converted_to_env + print("") + print("Please use script tags or alias/uid to get help for a specific script") + print("") + print("Generic Inputs for all Scripts:") + print("") + print_input_descriptions(generic_inputs) - # Step 4: Iterate over scripts and generate help output - for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): - metadata = script.meta - script_path = script.path - print_script_help( - metadata, - script_path, - generic_inputs, - env, - self_module) + else: + # Step 4: Iterate over scripts and generate help output + for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): + metadata = script.meta + script_path = script.path + print_script_help( + metadata, + script_path, + generic_inputs, + env, + self_module) return {'return': 0} diff --git a/automation/script/module.py b/automation/script/module.py index 827b7f8cc..298a6e2bc 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2870,6 +2870,8 @@ def search(self, i): if i.get(key): ii[key] = i[key] + if i.get('all'): + ii['all'] = i['all'] r = super(ScriptAutomation, self).search(ii) if r['return'] > 0: return r diff --git a/script/app-mlperf-inference-mlcommons-python/README.md b/script/app-mlperf-inference-mlcommons-python/README.md index adefcc231..58bf4738e 100644 --- a/script/app-mlperf-inference-mlcommons-python/README.md +++ b/script/app-mlperf-inference-mlcommons-python/README.md @@ -1,10 +1,10 @@ # README for app-mlperf-inference-mlcommons-python This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -136,7 +136,7 @@ mlcr app,vision,language,mlcommons,mlperf,inference,reference,ref - `llama2-70b-99` (base: llama2-70b_) - `llama2-70b-99.9` (base: llama2-70b_) - `llama3_1-405b` -- `llama3_1-8b` +- `llama3_1-8b_` - `mixtral-8x7b` - `pointpainting` - `resnet50` (default) @@ -166,6 +166,8 @@ mlcr app,vision,language,mlcommons,mlperf,inference,reference,ref - `dlrm-v2_` - `gptj_` - `llama2-70b_` +- `llama3_1-8b` (base: llama3_1-8b_) +- `llama3_1-8b-edge` (base: llama3_1-8b_) - `multistream` - `offline` - `r2.1_default` diff --git a/script/app-mlperf-inference-nvidia/README.md b/script/app-mlperf-inference-nvidia/README.md index 9075672d8..261cf91ca 100644 --- a/script/app-mlperf-inference-nvidia/README.md +++ b/script/app-mlperf-inference-nvidia/README.md @@ -1,13 +1,96 @@ # README for app-mlperf-inference-nvidia -This README is automatically generated. Add custom content in [info.txt](info.txt). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. ## Run Commands ```bash mlcr reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia ``` -No script specific inputs +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--count` | | | `` | +| `--max_batchsize` | | | `` | +| `--mlperf_conf` | | | `` | +| `--mode` | | | `performance` | +| `--output_dir` | | | `` | +| `--scenario` | | | `Offline` | +| `--user_conf` | | | `` | +| `--devices` | | | `` | +| `--skip_preprocess` | | | `no` | +| `--skip_preprocessing` | Alias for skip_preprocess | | `` | +| `--target_qps` | | | `` | +| `--offline_target_qps` | | | `` | +| `--server_target_qps` | | | `` | +| `--target_latency` | | | `` | +| `--singlestream_target_latency` | | | `` | +| `--multistream_target_latency` | | | `` | +| `--use_triton` | | | `` | +| `--gpu_copy_streams` | | | `` | +| `--gpu_inference_streams` | | | `` | +| `--gpu_batch_size` | | | `` | +| `--dla_copy_streams` | | | `` | +| `--dla_inference_streams` | | | `` | +| `--dla_batch_size` | | | `` | +| `--input_format` | | | `` | +| `--performance_sample_count` | | | `` | +| `--workspace_size` | | | `` | +| `--log_dir` | | | `` | +| `--use_graphs` | | | `` | +| `--run_infer_on_copy_streams` | | | `` | +| `--start_from_device` | | | `` | +| `--end_on_device` | | | `` | +| `--max_dlas` | | | `` | +| `--power_setting` | | | `` | +| `--make_cmd` | | | `` | +| `--rerun` | | | `` | +| `--extra_run_options` | | | `` | +| `--use_deque_limit` | | | `` | +| `--deque_timeout_usec` | | | `` | +| `--use_cuda_thread_per_device` | | | `` | +| `--num_warmups` | | | `` | +| `--graphs_max_seqlen` | | | `` | +| `--num_issue_query_threads` | | | `` | +| `--soft_drop` | | | `` | +| `--use_small_tile_gemm_plugin` | | | `` | +| `--audio_buffer_num_lines` | | | `` | +| `--use_fp8` | | | `` | +| `--enable_sort` | | | `` | +| `--num_sort_segments` | | | `` | +| `--skip_postprocess` | | | `` | +| `--embedding_weights_on_gpu_part` | | | `` | +| `--sdxl_batcher_time_limit` | | | `` | ### Generic Script Inputs | Name | Description | Choices | Default | @@ -26,3 +109,128 @@ No script specific inputs | `--gh_token` | Github Token | | `` | | `--hf_token` | Huggingface Token | | `` | | `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Backend + +- `tensorrt` (default) + +### Batch-size + +- `batch_size.#` _(# can be substituted dynamically)_ + +### Batchsize-format-change + +- `pre5.0` +- `v5.0+` + +### Build-engine-options + +- `build_engine_options.#` _(# can be substituted dynamically)_ + +### Device + +- `cpu` +- `cuda` (default) + +### Device-memory + +- `gpu_memory.#` _(# can be substituted dynamically)_ +- `gpu_memory.16` +- `gpu_memory.24` +- `gpu_memory.32` +- `gpu_memory.40` +- `gpu_memory.48` +- `gpu_memory.8` +- `gpu_memory.80` + +### Dla-batch-size + +- `dla_batch_size.#` _(# can be substituted dynamically)_ + +### Gpu-connection + +- `pcie` +- `sxm` + +### Gpu-name + +- `a100` +- `a6000` +- `custom` +- `l4` +- `orin` +- `rtx_4090` +- `rtx_6000_ada` +- `t4` + +### Graphs + +- `use-graphs` + +### Loadgen-scenario + +- `multistream` +- `offline` +- `server` +- `singlestream` + +### Model + +- `3d-unet-99` (base: 3d-unet_) +- `3d-unet-99.9` (base: 3d-unet_) +- `bert-99` (base: bert_) +- `bert-99.9` (base: bert_) +- `dlrm-v2-99` (base: dlrm_) +- `dlrm-v2-99.9` (base: dlrm_) +- `gptj-99` (base: gptj_) +- `gptj-99.9` (base: gptj_) +- `llama2-70b-99` (base: llama2-70b_) +- `llama2-70b-99.9` (base: llama2-70b_) +- `resnet50` (default) +- `retinanet` +- `rnnt` +- `sdxl` + +### Num-gpus + +- `num-gpus.#` _(# can be substituted dynamically)_ +- `num-gpus.1` (default) + +### Power-mode + +- `maxn` +- `maxq` + +### Run-mode + +- `build` +- `build_engine` (alias: build-engine) +- `calibrate` +- `download_model` +- `prebuild` +- `preprocess_data` (alias: preprocess-data) +- `run_harness` (default) + +### Triton + +- `use_triton` + +### Ungrouped + +- `3d-unet_` +- `bert_` +- `default_variations` +- `dlrm_` +- `env` +- `gptj_` +- `llama2-70b_` +- `run-harness` +- `v3.1` (base: pre5.0) + +### Version + +- `v4.0` (base: pre5.0) +- `v4.1` (base: pre5.0) +- `v4.1-dev` (base: pre5.0) (default) +- `v5.0` (base: v5.0+) diff --git a/script/app-mlperf-inference/README.md b/script/app-mlperf-inference/README.md index 6a5c63c29..6f8c726c9 100644 --- a/script/app-mlperf-inference/README.md +++ b/script/app-mlperf-inference/README.md @@ -1,10 +1,10 @@ # README for app-mlperf-inference This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -166,7 +166,7 @@ mlcr app,vision,language,mlcommons,mlperf,inference,generic - `llama2-70b-99` (base: llama2-70b_) - `llama2-70b-99.9` (base: llama2-70b_) - `llama3_1-405b` -- `llama3_1-8b` +- `llama3_1-8b_` - `mixtral-8x7b` (base: mixtral-8x7b) - `mobilenet` - `pointpainting` @@ -207,4 +207,6 @@ mlcr app,vision,language,mlcommons,mlperf,inference,generic - `dlrm_` - `gptj_` (alias: gptj) - `llama2-70b_` +- `llama3_1-8b` (base: llama3_1-8b_) +- `llama3_1-8b-edge` (base: llama3_1-8b_) - `power` diff --git a/script/build-mlperf-inference-server-nvidia/README.md b/script/build-mlperf-inference-server-nvidia/README.md index 280d286b0..0714b29f1 100644 --- a/script/build-mlperf-inference-server-nvidia/README.md +++ b/script/build-mlperf-inference-server-nvidia/README.md @@ -1,10 +1,10 @@ # README for build-mlperf-inference-server-nvidia This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/download-file/README.md b/script/download-file/README.md index 5caa972b5..dcc6f866c 100644 --- a/script/download-file/README.md +++ b/script/download-file/README.md @@ -1,10 +1,10 @@ # README for download-file This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -72,7 +72,7 @@ mlcr download file - `curl` - `gdown` - `mlcutil` (alias: cmutil) (default) -- `r2_downloader` +- `r2-downloader` - `rclone` - `wget` diff --git a/script/generate-mlperf-inference-submission/README.md b/script/generate-mlperf-inference-submission/README.md index 416a01828..603dc5d8d 100644 --- a/script/generate-mlperf-inference-submission/README.md +++ b/script/generate-mlperf-inference-submission/README.md @@ -1,10 +1,10 @@ # README for generate-mlperf-inference-submission This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/get-cache-dir/README.md b/script/get-cache-dir/README.md index 172b85eec..b9f650517 100644 --- a/script/get-cache-dir/README.md +++ b/script/get-cache-dir/README.md @@ -1,10 +1,10 @@ # README for get-cache-dir This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/get-generic-sys-util/README.md b/script/get-generic-sys-util/README.md index ec30d7828..45892d2fb 100644 --- a/script/get-generic-sys-util/README.md +++ b/script/get-generic-sys-util/README.md @@ -1,10 +1,10 @@ # README for get-generic-sys-util This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -72,6 +72,7 @@ mlcr get,sys-util,generic,generic-sys-util - `autoconf` - `bzip2` - `cmake` +- `crossbuild-essential-arm64` - `dmidecode` - `ffmpeg` - `flex` diff --git a/script/get-git-repo/README.md b/script/get-git-repo/README.md index 9c68729c1..f79fe3b3a 100644 --- a/script/get-git-repo/README.md +++ b/script/get-git-repo/README.md @@ -1,10 +1,10 @@ # README for get-git-repo This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/get-ipol-src/meta.yaml b/script/get-ipol-src/meta.yaml index ac64d8971..92e388799 100644 --- a/script/get-ipol-src/meta.yaml +++ b/script/get-ipol-src/meta.yaml @@ -13,8 +13,10 @@ extra_cache_tags_from_env: - env: MLC_IPOL_YEAR prefix: year- input_description: - number: IPOL publication number - year: IPOL publication year + number: + desc: IPOL publication number + year: + desc: IPOL publication year input_mapping: number: MLC_IPOL_NUMBER year: MLC_IPOL_YEAR diff --git a/script/get-lib-jemalloc/README.md b/script/get-lib-jemalloc/README.md index f61ba4247..f6b90d0f5 100644 --- a/script/get-lib-jemalloc/README.md +++ b/script/get-lib-jemalloc/README.md @@ -1,6 +1,35 @@ # README for get-lib-jemalloc -This README is automatically generated. Add custom content in [info.txt](info.txt). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. ## Run Commands ```bash @@ -26,3 +55,21 @@ No script specific inputs | `--gh_token` | Github Token | | `` | | `--hf_token` | Huggingface Token | | `` | | `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Ungrouped + +- `branch.#` _(# can be substituted dynamically)_ +- `config.#` _(# can be substituted dynamically)_ +- `enable-prof` +- `enable-stats` +- `lg-hugepage.#` _(# can be substituted dynamically)_ +- `lg-page.#` _(# can be substituted dynamically)_ +- `lg-quantum.#` _(# can be substituted dynamically)_ +- `sha.#` _(# can be substituted dynamically)_ +- `version.official` (base: url.official) + +### Version + +- `url.#` _(# can be substituted dynamically)_ +- `url.official` (default) diff --git a/script/get-ml-model-abtf-ssd-pytorch/README.md b/script/get-ml-model-abtf-ssd-pytorch/README.md index 3ef5cbbdc..6a6e5d631 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/README.md +++ b/script/get-ml-model-abtf-ssd-pytorch/README.md @@ -1,10 +1,10 @@ # README for get-ml-model-abtf-ssd-pytorch This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/get-ml-model-deeplabv3_plus/README.md b/script/get-ml-model-deeplabv3_plus/README.md index 90f6525bc..438f35648 100644 --- a/script/get-ml-model-deeplabv3_plus/README.md +++ b/script/get-ml-model-deeplabv3_plus/README.md @@ -1,10 +1,10 @@ # README for get-ml-model-deeplabv3-plus This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/get-ml-model-resnet50/README.md b/script/get-ml-model-resnet50/README.md index d6e3dd61f..13012c78b 100644 --- a/script/get-ml-model-resnet50/README.md +++ b/script/get-ml-model-resnet50/README.md @@ -1,10 +1,10 @@ # README for get-ml-model-resnet50 This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. diff --git a/script/get-oneapi/README.md b/script/get-oneapi/README.md index 7d8dedc9c..84cc32210 100644 --- a/script/get-oneapi/README.md +++ b/script/get-oneapi/README.md @@ -1,13 +1,46 @@ # README for get-one-api -This README is automatically generated. Add custom content in [info.txt](info.txt). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do +``` +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC +``` +You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. + +## Setup + +If you are not on a Python development environment please refer to the [official docs](https://docs.mlcommons.org/mlcflow/install/) for the installation. + +```bash +python3 -m venv mlcflow +. mlcflow/bin/activate +pip install mlcflow +``` + +- Using a virtual environment is recommended (per `pip` best practices), but you may skip it or use `--break-system-packages` if needed. + +### Pull mlperf-automations + +Once `mlcflow` is installed: + +```bash +mlc pull repo mlcommons@mlperf-automations --pat= +``` +- `--pat` or `--ssh` is only needed if the repo is PRIVATE +- If `--pat` is avoided, you'll be asked to enter the password where you can enter your Private Access Token +- `--ssh` option can be used instead of `--pat=<>` option if you prefer to use SSH for accessing the github repository. ## Run Commands ```bash mlcr get,oneapi,compiler,get-oneapi ``` -No script specific inputs +### Script Inputs + +| Name | Description | Choices | Default | +|------|-------------|---------|------| +| `--oneapi_dir` | | | `` | ### Generic Script Inputs | Name | Description | Choices | Default | @@ -26,3 +59,9 @@ No script specific inputs | `--gh_token` | Github Token | | `` | | `--hf_token` | Huggingface Token | | `` | | `--verify_ssl` | Verify SSL | | `False` | +## Variations + +### Ungrouped + +- `fortran` +- `path.#` _(# can be substituted dynamically)_ diff --git a/script/get-spec-ptd/meta.yaml b/script/get-spec-ptd/meta.yaml index f642f7053..8c6087600 100644 --- a/script/get-spec-ptd/meta.yaml +++ b/script/get-spec-ptd/meta.yaml @@ -24,7 +24,8 @@ deps: - MLC_GIT_* tags: get,git,repo,_repo.https://github.com/mlcommons/power input_description: - input: Path to SPEC PTDaemon (Optional) + input: + desc: Path to SPEC PTDaemon (Optional) input_mapping: input: MLC_INPUT new_env_keys: diff --git a/script/get-tensorrt/meta.yaml b/script/get-tensorrt/meta.yaml index 5370e37d4..1263e4158 100644 --- a/script/get-tensorrt/meta.yaml +++ b/script/get-tensorrt/meta.yaml @@ -13,9 +13,10 @@ deps: tags: get,python3 docker: {} input_description: - input: Full path to the installed TensorRT library (nvinfer) - tar_file: Full path to the TensorRT Tar file downloaded from the Nvidia website - (https://developer.nvidia.com/tensorrt) + input: + desc: Full path to the installed TensorRT library (nvinfer) + tar_file: + desc: Full path to the TensorRT Tar file downloaded from the Nvidia website (https://developer.nvidia.com/tensorrt) input_mapping: input: MLC_INPUT tar_file: MLC_TENSORRT_TAR_FILE_PATH diff --git a/script/run-mlperf-inference-submission-checker/README.md b/script/run-mlperf-inference-submission-checker/README.md index 626aa9fcd..ce18e987b 100644 --- a/script/run-mlperf-inference-submission-checker/README.md +++ b/script/run-mlperf-inference-submission-checker/README.md @@ -1,10 +1,10 @@ # README for run-mlperf-inference-submission-checker This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. @@ -90,3 +90,7 @@ mlcr run,mlc,mlcommons,mlperf,inference,mlperf-inference,submission,checker,subm ### Ungrouped - `short-run` + +### Version + +- `version.master` diff --git a/script/submit-mlperf-results/README.md b/script/submit-mlperf-results/README.md index 6a8108ee0..f1909fca1 100644 --- a/script/submit-mlperf-results/README.md +++ b/script/submit-mlperf-results/README.md @@ -1,10 +1,10 @@ # README for submit-mlperf-results This README is automatically generated. Add custom content in [info.md](info.md). Please follow the [script execution document](https://docs.mlcommons.org/mlcflow/targets/script/execution-flow/) to understand more about the MLC script execution. -`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/user`, you can do +`mlcflow` stores all local data under `$HOME/MLC` by default. So, if there is space constraint on the home directory and you have more space on say `/mnt/$USER`, you can do ``` -mkdir /mnt/user/MLC -ln -s /mnt/user/MLC $HOME/MLC +mkdir /mnt/$USER/MLC +ln -s /mnt/$USER/MLC $HOME/MLC ``` You can also use the `ENV` variable `MLC_REPOS` to control this location but this will need a set after every system reboot. From 812f21154a4fb7041216e15a23c5c3cda440b5fe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 17 Aug 2025 16:46:26 +0000 Subject: [PATCH 116/124] [Automated Commit] Format Codebase [skip ci] --- automation/script/help.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automation/script/help.py b/automation/script/help.py index f4d9ab3e5..044052ce5 100644 --- a/automation/script/help.py +++ b/automation/script/help.py @@ -42,7 +42,8 @@ def display_help(self_module, input_params): else: # Step 4: Iterate over scripts and generate help output - for script in sorted(scripts_list, key=lambda x: x.meta.get('alias', '')): + for script in sorted( + scripts_list, key=lambda x: x.meta.get('alias', '')): metadata = script.meta script_path = script.path print_script_help( From d945cbeb1e9c7307de742719713de624b6ce0859 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 21 Aug 2025 03:56:25 +0530 Subject: [PATCH 117/124] Removed protobuf version requirement for mlperf-inference resnet50 --- script/app-mlperf-inference-mlcommons-python/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/app-mlperf-inference-mlcommons-python/meta.yaml b/script/app-mlperf-inference-mlcommons-python/meta.yaml index 13415df06..3fdff095d 100644 --- a/script/app-mlperf-inference-mlcommons-python/meta.yaml +++ b/script/app-mlperf-inference-mlcommons-python/meta.yaml @@ -1322,8 +1322,8 @@ variations: - tags: get,generic-python-lib,_protobuf names: - protobuf - version_max: "4.23.4" - version_max_usable: "4.23.4" + version_max1: "4.23.4" + version_max_usable1: "4.23.4" version_min: "3.20.3" enable_if_env: MLC_MLPERF_BACKEND: From 8a5ac93d2f4c896e87d41b32e8367bc8804255f2 Mon Sep 17 00:00:00 2001 From: Arjun Date: Mon, 25 Aug 2025 01:27:40 +0530 Subject: [PATCH 118/124] Support oneapi 2025.2.1 --- script/get-oneapi/install.sh | 4 ++-- script/get-oneapi/meta.yaml | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/script/get-oneapi/install.sh b/script/get-oneapi/install.sh index 54f9178b2..a299a37f3 100644 --- a/script/get-oneapi/install.sh +++ b/script/get-oneapi/install.sh @@ -28,8 +28,8 @@ CMD="bash ./${MLC_ONEAPI_INSTALL_FILENAME} -a --silent --cli --eula accept --in run "$CMD" if [[ ${MLC_ONEAPI_FORTRAN} == 'yes' ]] then - wget -nc https://registrationcenter-download.intel.com/akdlm/IRC_NAS/2238465b-cfc7-4bf8-ad04-e55cb6577cba/intel-fortran-essentials-2025.1.1.8_offline.sh + wget -nc ${MLC_ONEAPI_FORTRAN_COMPILER_URL_BASE}/${MLC_ONEAPI_FORTRAN_COMPILER_FILENAME} test $? -eq 0 || exit $? - CMD="bash ./intel-fortran-essentials-2025.1.1.8_offline.sh -a --silent --cli --eula accept --install-dir ${PWD}/install" + CMD="bash ./${MLC_ONEAPI_FORTRAN_COMPILER_FILENAME} -a --silent --cli --eula accept --install-dir ${PWD}/install" run "$CMD" fi diff --git a/script/get-oneapi/meta.yaml b/script/get-oneapi/meta.yaml index 5f9ad5b1f..c0eaee41b 100644 --- a/script/get-oneapi/meta.yaml +++ b/script/get-oneapi/meta.yaml @@ -35,6 +35,15 @@ tests: - fortran versions: + 2025.2.1: + env: + MLC_ONEAPI_INSTALL_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/3b7a16b3-a7b0-460f-be16-de0d64fa6b1e + MLC_ONEAPI_INSTALL_FILENAME: intel-oneapi-base-toolkit-2025.2.1.44_offline.sh + MLC_ONEAPI_FORTRAN_COMPILER_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/a31a1a8e-4b6f-4548-a554-fe3ac1c8b230 + MLC_ONEAPI_FORTRAN_COMPILER_FILENAME: intel-fortran-essentials-2025.2.1.5_offline.sh + MLC_ONEAPI_INSTALL_VERSION_PREFIX: '2025.2' + MLC_VERSION: '2025.2.1' + 2025.2.0: env: MLC_ONEAPI_INSTALL_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/bd1d0273-a931-4f7e-ab76-6a2a67d646c7 @@ -46,6 +55,8 @@ versions: env: MLC_ONEAPI_INSTALL_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/6bfca885-4156-491e-849b-1cd7da9cc760 MLC_ONEAPI_INSTALL_FILENAME: intel-oneapi-base-toolkit-2025.1.1.36_offline.sh + MLC_ONEAPI_FORTRAN_COMPILER_URL_BASE: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/2238465b-cfc7-4bf8-ad04-e55cb6577cba + MLC_ONEAPI_FORTRAN_COMPILER_FILENAME: intel-fortran-essentials-2025.1.1.8_offline.sh MLC_ONEAPI_INSTALL_VERSION_PREFIX: '2025.1' MLC_VERSION: '2025.1.1' From 2d16dce778e0e990ef912e8262526a117f06b98e Mon Sep 17 00:00:00 2001 From: Arjun Date: Mon, 25 Aug 2025 02:14:14 +0530 Subject: [PATCH 119/124] Support oneapi 2025.2.1 --- script/get-oneapi/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-oneapi/meta.yaml b/script/get-oneapi/meta.yaml index c0eaee41b..cec4b7aa5 100644 --- a/script/get-oneapi/meta.yaml +++ b/script/get-oneapi/meta.yaml @@ -4,7 +4,7 @@ automation_uid: 5b4e0237da074764 cache: true category: Compiler automation clean_files: [] -default_version: 2025.2.0 +default_version: 2025.2.1 deps: - tags: detect,os name: Detect or install OneAPI compiler From b4301c0f44ba0e8b06337de8beddd8b7c3509b0d Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 3 Sep 2025 05:40:49 +0100 Subject: [PATCH 120/124] Support install-gcc-src on macos --- script/install-gcc-src/customize.py | 7 ++++++- script/install-gcc-src/meta.yaml | 17 ++++++++++++++--- script/install-gcc-src/run.sh | 4 +++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index c13141d22..8bcdfb7aa 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -23,10 +23,15 @@ def preprocess(i): env['MLC_GCC_TARGET_STRING'] = '' if env.get('MLC_GCC_HOST', '') != '': - env['MLC_GCC_HOST_STRING'] = f""" --target={env['MLC_GCC_HOST']} """ + env['MLC_GCC_HOST_STRING'] = f""" --host={env['MLC_GCC_HOST']} """ else: env['MLC_GCC_HOST_STRING'] = '' + if env.get('MLC_GCC_BUILD', '') != '': + env['MLC_GCC_BUILD_STRING'] = f""" --build={env['MLC_GCC_BUILD']} """ + else: + env['MLC_GCC_BUILD_STRING'] = '' + env['MLC_GCC_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') return {'return': 0} diff --git a/script/install-gcc-src/meta.yaml b/script/install-gcc-src/meta.yaml index 3bc1e7083..138d7f86e 100644 --- a/script/install-gcc-src/meta.yaml +++ b/script/install-gcc-src/meta.yaml @@ -26,10 +26,11 @@ deps: - MLC_GIT_CHECKOUT_TAG input_mapping: targets: MLC_GCC_TARGET + target: MLC_GCC_TARGET host: MLC_GCC_HOST + build: MLC_GCC_BUILD + extra: MLC_GCC_EXTRA_CONFIGURE_STRING -env: - MLC_GIT_URL: git://gcc.gnu.org/git/gcc.git new_env_keys: - MLC_GCC_* @@ -52,9 +53,19 @@ tests: run_inputs: - {} variations: + mainline: + group: repo + default: true + default_variations: + version: master + env: + MLC_GIT_URL: git://gcc.gnu.org/git/gcc.git + darwin: + group: repo + env: + MLC_GIT_URL: https://github.com/iains/gcc-darwin-arm64 master: group: version - default: true env: MLC_GIT_CHECKOUT: master version.#: diff --git a/script/install-gcc-src/run.sh b/script/install-gcc-src/run.sh index d11bedbd0..9bc0a5eea 100644 --- a/script/install-gcc-src/run.sh +++ b/script/install-gcc-src/run.sh @@ -11,6 +11,8 @@ if [ ! -d "src" ]; then test $? -eq 0 || exit $? fi +rm -rf install +rm -rf build mkdir -p install mkdir -p build @@ -23,7 +25,7 @@ cd src cd ../build -cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} ${MLC_GCC_HOST_STRING} --with-gcc-major-version-only --disable-multilib" +cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} ${MLC_GCC_HOST_STRING} ${MLC_GCC_BUILD_STRING} ${MLC_GCC_EXTRA_CONFIGURE_STRING} --with-gcc-major-version-only --disable-multilib" echo $cmd eval $cmd From 37f8f3563468aaa3902baae0d8a515b3bc067920 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 3 Sep 2025 16:49:38 +0530 Subject: [PATCH 121/124] Add --with-sysroot by default for gcc install on macos --- script/install-gcc-src/customize.py | 5 +++++ script/install-gcc-src/meta.yaml | 1 + script/install-gcc-src/run.sh | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/script/install-gcc-src/customize.py b/script/install-gcc-src/customize.py index 8bcdfb7aa..576e180bc 100644 --- a/script/install-gcc-src/customize.py +++ b/script/install-gcc-src/customize.py @@ -32,6 +32,11 @@ def preprocess(i): else: env['MLC_GCC_BUILD_STRING'] = '' + if env.get('MLC_GCC_WITH_SYSROOT', '') != '': + env['MLC_GCC_SYSROOT_STRING'] = f""" --with-sysroot={env['MLC_GCC_WITH_SYSROOT']} """ + else: + env['MLC_GCC_SYSROOT_STRING'] = '' + env['MLC_GCC_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin') return {'return': 0} diff --git a/script/install-gcc-src/meta.yaml b/script/install-gcc-src/meta.yaml index 138d7f86e..9380843ca 100644 --- a/script/install-gcc-src/meta.yaml +++ b/script/install-gcc-src/meta.yaml @@ -64,6 +64,7 @@ variations: group: repo env: MLC_GIT_URL: https://github.com/iains/gcc-darwin-arm64 + MLC_GCC_WITH_SYSROOT: '`xcrun --show-sdk-path`' master: group: version env: diff --git a/script/install-gcc-src/run.sh b/script/install-gcc-src/run.sh index 9bc0a5eea..bf0a78b39 100644 --- a/script/install-gcc-src/run.sh +++ b/script/install-gcc-src/run.sh @@ -25,7 +25,7 @@ cd src cd ../build -cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} ${MLC_GCC_HOST_STRING} ${MLC_GCC_BUILD_STRING} ${MLC_GCC_EXTRA_CONFIGURE_STRING} --with-gcc-major-version-only --disable-multilib" +cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} ${MLC_GCC_HOST_STRING} ${MLC_GCC_BUILD_STRING} ${MLC_GCC_SYSROOT_STRING} ${MLC_GCC_EXTRA_CONFIGURE_STRING} --with-gcc-major-version-only --disable-multilib" echo $cmd eval $cmd From 2ebd1938ebbf2929dcb4a0141421088b61fceda1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 3 Sep 2025 17:14:46 +0530 Subject: [PATCH 122/124] --target not allowed as script input --- script/install-gcc-src/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/script/install-gcc-src/meta.yaml b/script/install-gcc-src/meta.yaml index 9380843ca..8879d3645 100644 --- a/script/install-gcc-src/meta.yaml +++ b/script/install-gcc-src/meta.yaml @@ -26,7 +26,6 @@ deps: - MLC_GIT_CHECKOUT_TAG input_mapping: targets: MLC_GCC_TARGET - target: MLC_GCC_TARGET host: MLC_GCC_HOST build: MLC_GCC_BUILD extra: MLC_GCC_EXTRA_CONFIGURE_STRING From effada3b9dc1be90f8e5fea24a22a7eb865c48b4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 3 Sep 2025 21:10:03 +0100 Subject: [PATCH 123/124] Create codeql.yml --- .github/workflows/codeql.yml | 102 +++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..03e2e3126 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,102 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Advanced" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '39 13 * * 1' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: actions + build-mode: none + - language: c-cpp + build-mode: autobuild + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Add any setup steps before running the `github/codeql-action/init` action. + # This includes steps like installing compilers or runtimes (`actions/setup-node` + # or others). This is typically only required for manual builds. + # - name: Setup runtime (example) + # uses: actions/setup-example@v1 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # โ„น๏ธ Command-line programs to run using the OS shell. + # ๐Ÿ“š See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + shell: bash + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From 1bacc430d8b56675df76cb1c0ae92452682fcafd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 3 Sep 2025 23:53:43 +0100 Subject: [PATCH 124/124] Update codeql.yml --- .github/workflows/codeql.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 03e2e3126..33b3ec5bb 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,14 +1,3 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "CodeQL Advanced" on: @@ -45,8 +34,6 @@ jobs: include: - language: actions build-mode: none - - language: c-cpp - build-mode: autobuild - language: python build-mode: none # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift'