Merge Dev

.github/workflows/document-scripts.yml

@@ -56,11 +56,6 @@ jobs:
         with:
           fetch-depth: 0
           path: automation-scripts
-
-      - name: Set up Git for commit
-        run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
 
       - name: Document meta.yaml file
         run: |
@@ -73,8 +68,8 @@ jobs:
           cd automation-scripts
           find . -type f -name README.md -exec git add {} +
           # Use the GitHub actor's name and email
-          git config --global user.name github-actions[bot]
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config user.name github-actions[bot]
+          git config user.email "github-actions[bot]@users.noreply.github.com"
           # Commit changes
           git diff-index --quiet HEAD || (git commit -am "[Automated Commit] Document ${{ matrix.modified_metas.file}}  [skip ci]" && git push)
 

automation/script/module.py

@@ -809,14 +809,15 @@ def _run(self, i):
             run_state['script_entry_repo_git'] = script_item.repo.meta.get(
                 'git', False)
 
-        deps = meta.get('deps', [])
-        post_deps = meta.get('post_deps', [])
-        prehook_deps = meta.get('prehook_deps', [])
-        posthook_deps = meta.get('posthook_deps', [])
+        deps = []
+        post_deps = []
+        prehook_deps = []
+        posthook_deps = []
         input_mapping = meta.get('input_mapping', {})
+        new_env_keys_from_meta = []
+        new_state_keys_from_meta = []
+
         docker_settings = meta.get('docker')
-        new_env_keys_from_meta = meta.get('new_env_keys', [])
-        new_state_keys_from_meta = meta.get('new_state_keys', [])
 
         found_script_item = utils.assemble_object(
             meta['alias'], meta['uid'])
@@ -844,22 +845,30 @@ def _run(self, i):
         for key in script_item_default_env:
             env.setdefault(key, script_item_default_env[key])
 
-        # Force env from meta['env'] as a CONST
-        # (env OVERWRITE)
-        script_item_env = meta.get('env', {})
-        # print(f"script meta env= {script_item_env}")
+        # for update_meta_if_env
 
-        utils.merge_dicts({'dict1': env,
-                           'dict2': script_item_env,
-                           'append_lists': True,
-                           'append_unique': True})
-        # print(f"env = {env}")
+        r = update_state_from_meta(
+            meta,
+            env,
+            state,
+            const,
+            const_state,
+            deps,
+            post_deps,
+            prehook_deps,
+            posthook_deps,
+            new_env_keys_from_meta,
+            new_state_keys_from_meta,
+            run_state,
+            i)
+        if r['return'] > 0:
+            return r
 
-        script_item_state = meta.get('state', {})
-        utils.merge_dicts({'dict1': state,
-                           'dict2': script_item_state,
-                           'append_lists': True,
-                           'append_unique': True})
+        # taking from meta or else deps with same names will be ignored
+        deps = meta.get('deps', [])
+        post_deps = meta.get('post_deps', [])
+        prehook_deps = meta.get('prehook_deps', [])
+        posthook_deps = meta.get('posthook_deps', [])
 
         # Store the default_version in run_state -> may be overridden by
         # variations
@@ -5604,7 +5613,7 @@ def convert_env_to_script(env, os_info, start_script=None):
                 os_info['env_var'].replace(
                     'env_var', key)}"""
 
-        env_quote = os_info['env_quote']
+        env_quote = os_info.get('env_quote', '"')
         # Replace placeholders in the platform-specific environment command
         # and escapes any quote in the env value
         env_command = os_info['set_env'].replace(

script/app-mlperf-inference-nvidia/README.md

@@ -234,3 +234,4 @@ mlcr reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia
 - `v4.1` (base: pre5.0)
 - `v4.1-dev` (base: pre5.0) (default)
 - `v5.0` (base: v5.0+)
+- `v5.1` (base: v5.0+)

script/app-mlperf-inference-nvidia/customize.py

@@ -346,7 +346,7 @@ def preprocess(i):
                 cmds.append(
                     f"mkdir -p {os.path.dirname(preprocessed_data_for_accuracy_checker)}")
             cmds.append(
-                f"ln -sf {env['MLC_DATASET_OPENORCA_PREPROCESSED_PATH']} {preprocessed_data_for_accuracy_checker}")
+                f"cp {env['MLC_DATASET_OPENORCA_PREPROCESSED_PATH']} {preprocessed_data_for_accuracy_checker}")
 
         model_name = "llama2-70b"
         model_path = fp8_model_path
@@ -361,7 +361,7 @@ def preprocess(i):
             else:
                 cmds.append(f"make download_model BENCHMARKS='{model_name}'")
         elif "stable-diffusion" in env['MLC_MODEL']:
-            if env.get('MLC_MLPERF_INFERENCE_CODE_VERSION') == 'v5.0':
+            if is_true(env.get('MLC_MLPERF_INFERENCE_POST_5_0')):
                 # Define folder mappings for each model type
                 model_folders = {
                     'onnx_models': ["clip1", "clip2", "unetxl", "vae"],
@@ -440,7 +440,7 @@ def preprocess(i):
                     f"mkdir -p {os.path.dirname(target_preprocessed_data_path)}")
                 if env.get('MLC_DATASET_OPENORCA_PREPROCESSED_PATH'):
                     cmds.append(
-                        f"ln -sf {env['MLC_DATASET_OPENORCA_PREPROCESSED_PATH']} {os.path.join(env['MLPERF_SCRATCH_PATH'], "preprocessed_data", "open_orca")}"
+                        f"cp -r {env['MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH']}/* {os.path.join(env['MLPERF_SCRATCH_PATH'], 'preprocessed_data', 'open_orca')}"
                     )
                 else:
                     cmds.append(
@@ -697,7 +697,8 @@ def preprocess(i):
 
         if "llama2" in env["MLC_MODEL"]:
             run_config += f" --checkpoint_dir={fp8_model_path}"
-            if env.get('MLC_MLPERF_INFERENCE_POST_5_0'):
+            run_config += f" --tensor_path={os.path.join(env['MLPERF_SCRATCH_PATH'], 'preprocessed_data', 'open_orca')}"
+            if is_true(env.get('MLC_MLPERF_INFERENCE_POST_5_0')):
                 run_config += f" --trtllm_build_flags=tensor_parallelism:{tmp_tp_size},pipeline_parallelism:{tmp_pp_size}"
             else:
                 run_config += f" --tensor_parallelism={tmp_tp_size}"

script/app-mlperf-inference-nvidia/meta.yaml

@@ -289,6 +289,8 @@ deps:
         - 'yes'
 
   - tags: get,nvidia,mitten
+    names:
+      - nvidia-mitten
     enable_if_env:
       MLC_NVIDIA_MITTEN_FROM_SRC:
         - 'yes'
@@ -357,6 +359,16 @@ variations:
     group: batchsize-format-change
     env:
       MLC_MLPERF_INFERENCE_POST_5_0: "yes"
+  v5.1:
+    base:
+      - v5.0+
+    group: version
+    env:
+      MLC_MLPERF_INFERENCE_CODE_VERSION: "v5.1"
+      MLC_NVIDIA_MITTEN_FROM_SRC: "yes"
+    adr:
+      pycuda:
+        version: "2024.1"
   v5.0:
     base:
       - v5.0+
@@ -786,6 +798,7 @@ variations:
       - tags: get,generic-python-lib,_package.rouge-score
         names:
           - rouge-score
+      - tags: get,generic-python-lib,_package.typeguard
     env:
       MLC_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://github.com/mlcommons/cm4mlops/blob/b18ff890ff559e21d2e27a3b54cd26467ac1fd9e/script/get-ml-model-llama2/_cm.json#L51"
       MLC_ML_MODEL_INPUTS_DATA_TYPE: int32

script/app-mlperf-inference/meta.yaml

@@ -463,6 +463,35 @@ variations:
         docker:
           base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v5.0-cuda12.8-pytorch25.01-ubuntu24.04-aarch64-Grace-release
 
+  nvidia-original,r5.1_default:
+    env:
+      MLC_NVIDIA_MITTEN_FROM_SRC: 'yes'
+    docker:
+      os_version: "24.04"
+      user: 'ubuntu'
+      build_deps:
+        - tags: detect,os
+      image_name: mlperf-inference-nvidia-v5.1-common
+      build_env:
+        ENV: release
+    deps:
+    - names:
+      - numpy
+      tags: get,generic-python-lib,_package.numpy
+      version_max: "1.26.999"
+      version_max_usable: "1.26.4"
+    update_meta_if_env:
+      - enable_if_env:
+          MLC_HOST_PLATFORM_FLAVOR:
+            - x86_64
+        docker:
+          base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v5.1-cuda12.9-pytorch25.05-ubuntu24.04-x86_64
+      - skip_if_env:
+          MLC_HOST_PLATFORM_FLAVOR:
+            - x86_64
+        docker:
+          base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v5.1-cuda12.9-pytorch25.05-ubuntu24.04-aarch64-Grace
+
   nvidia-original,gptj_:
     env:
       BUILD_TRTLLM: 1
@@ -578,6 +607,24 @@ variations:
         - tags: get,dataset,preprocessed,openorca,_validation,_mlcommons,_nvidia
     env:
       BUILD_TRTLLM: 1
+
+  nvidia-original,r5.1_default,llama2-70b_:
+    default_variations:
+      precision: float8
+    docker:
+      deps:
+        - tags: get,ml-model,llama2-70b,_nvidia,_fp8,_v5.1
+          names:
+            - llama2-model  
+          update_tags_from_env_with_prefix:
+            _tp-size.:
+              - MLC_NVIDIA_TP_SIZE
+            _pp-size.:
+              - MLC_NVIDIA_PP_SIZE
+        - tags: get,dataset,preprocessed,openorca,_calibration,_mlcommons,_nvidia
+        - tags: get,dataset,preprocessed,openorca,_validation,_mlcommons,_nvidia
+    env:
+      BUILD_TRTLLM: 1
 
   nvidia-original:
     docker:
@@ -1825,7 +1872,7 @@ variations:
       mlperf-inference-implementation:
         tags: _cuda
     deps:
-      - tags: get,cuda-devices,_with-pycuda
+      - tags: get,cuda-devices
         skip_if_env:
           MLC_CUDA_DEVICE_PROP_GLOBAL_MEMORY:
             - "yes"
@@ -2227,6 +2274,32 @@ variations:
       MLC_REGENERATE_MEASURE_FILES: 'yes'
       MLC_MLPERF_INFERENCE_VERSION: '5.1'
 
+  r5.1_default:
+    group:
+      reproducibility
+    add_deps_recursive:
+      nvidia-inference-common-code:
+        tags: _mlcommons,_v5.1
+      nvidia-inference-server:
+        version: r5.1
+        tags: _mlcommons
+      nvidia-harness:   
+        tags: _v5.1
+      intel-harness:
+        tags: _v4.1
+      nvidia-scratch-space:
+        tags: _version.5.1
+      nvidia-mitten-git-src:
+        tags: _sha.69fc0e5042ff1acdd0862e13d834d5b1d12e917b
+      nvidia-mitten:
+        tags: _default
+      pycuda:
+        version_min: "2024.1"
+    default_env:
+      MLC_SKIP_SYS_UTILS: 'yes'
+      MLC_REGENERATE_MEASURE_FILES: 'yes'
+      MLC_MLPERF_INFERENCE_VERSION: '5.1'
+
 invalid_variation_combinations:
   -
     - retinanet
@@ -2252,6 +2325,9 @@ invalid_variation_combinations:
   -
     - gptj
     - tf
+  -
+    - gptj
+    - r5.1_default
 
 input_description:
   scenario: 
@@ -2367,6 +2443,7 @@ docker:
    - "${{ LLAMA2_CHECKPOINT_PATH }}:${{ LLAMA2_CHECKPOINT_PATH }}"
    - "${{ LLAMA2_PRE_QUANTIZED_CHECKPOINT_PATH }}:${{ LLAMA2_PRE_QUANTIZED_CHECKPOINT_PATH }}"
    - "${{ MLC_DATASET_OPENORCA_PREPROCESSED_PATH }}:${{ MLC_DATASET_OPENORCA_PREPROCESSED_PATH }}"
+   - "${{ MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH }}:${{ MLC_DATASET_OPENORCA_NVIDIA_PREPROCESSED_PATH }}"
    - "${{ MLC_DATASET_OPENORCA_CALIBRATION_PATH }}:${{ MLC_DATASET_OPENORCA_CALIBRATION_PATH }}"
    - "${{ MLC_NVIDIA_LLAMA_DATASET_FILE_PATH }}:${{ MLC_NVIDIA_LLAMA_DATASET_FILE_PATH }}"
    - "${{ SDXL_CHECKPOINT_PATH }}:${{ SDXL_CHECKPOINT_PATH }}"

script/build-mlperf-inference-server-nvidia/meta.yaml

@@ -78,7 +78,7 @@ deps:
 
   # Detect MLCake
   - tags: get,cmake
-    version: "3.26.4"
+    version: "3.27"
 
   # Detect Google Logger
   - tags: get,generic,sys-util,_glog-dev
@@ -238,6 +238,14 @@ variations:
       numpy:
         version_max: "1.26.999"
         version_max_usable: "1.26.4"
+
+  r5.1:
+    add_deps_recursive:
+      pycuda:
+        version: "2024.1"
+      numpy:
+        version_max: "1.26.999"
+        version_max_usable: "1.26.4"
 
 
 versions:

script/download-and-extract/meta.yaml

@@ -62,12 +62,14 @@ tags:
 tags_help: download-and-extract file
 uid: c67e81a4ce2649f5
 variations:
-  cmutil:
+  mlcutil:
     add_deps:
       download-script:
-        tags: _cmutil
+        tags: _mlcutil
     default: true
     group: download-tool
+  cmutil:
+    alias: mlcutil
   curl:
     add_deps:
       download-script:

script/download-file/customize.py

@@ -204,6 +204,8 @@ def preprocess(i):
             logger.info(f"{env['MLC_DOWNLOAD_CMD']}")
 
         elif tool == "r2-downloader":
+            if is_true(env.get('MLC_AUTH_USING_SERVICE_ACCOUNT')):
+                extra_download_options += " -s "
             env['MLC_DOWNLOAD_CMD'] = f"bash <(curl -s https://raw.githubusercontent.com/mlcommons/r2-downloader/refs/heads/main/mlc-r2-downloader.sh) "
             if env["MLC_HOST_OS_TYPE"] == "windows":
                 # have to modify the variable from url to temp_url if it is