Changes for supporting Llama2-70b - r5.0

script/app-mlperf-inference-nvidia/customize.py

@@ -321,15 +321,30 @@ def preprocess(i):
         # path to which the data file is present
         target_data_path = os.path.join(
             env['MLPERF_SCRATCH_PATH'],
-            'preprocessed_data',
-            'open_orca')
+            'data',
+            'llama2-70b')
         # path to the dataset file
         target_data_file_path = os.path.join(
+            env['MLPERF_SCRATCH_PATH'],
+            'data',
+            'llama2-70b',
+            'open_orca_gpt4_tokenized_llama.sampled_24576.pkl')
+
+        preprocessed_data_for_accuracy_checker = os.path.join(
             env['MLPERF_SCRATCH_PATH'],
             'preprocessed_data',
             'open_orca',
             'open_orca_gpt4_tokenized_llama.sampled_24576.pkl')
+
+        if not env.get('LLAMA2_PRE_QUANTIZED_CHECKPOINT_PATH'):
+            target_calibration_data_file_path = os.path.join(
+                env['MLPERF_SCRATCH_PATH'],
+                'data',
+                'llama2-70b',
+                'open_orca_gpt4_tokenized_llama.calibration_1000.pkl')
+
         tmp_tp_size = env['MLC_NVIDIA_TP_SIZE']
+        tmp_pp_size = env['MLC_NVIDIA_PP_SIZE']
         if tmp_tp_size == "1":
             fp8_model_path = os.path.join(
                 env['MLPERF_SCRATCH_PATH'],
@@ -343,15 +358,35 @@ def preprocess(i):
                 'models',
                 'Llama2',
                 'fp8-quantized-ammo',
-                f'llama2-70b-chat-hf-tp{tmp_tp_size}pp1-fp8')
+                f'llama2-70b-chat-hf-tp{tmp_tp_size}pp{tmp_pp_size}-fp8')
+
+        # check the presence of validation dataset
         if not os.path.exists(target_data_file_path):
-            if env.get('MLC_NVIDIA_LLAMA_DATASET_FILE_PATH', '') == '':
+            if env.get('MLC_DATASET_OPENORCA_PREPROCESSED_PATH', '') == '':
                 return {
-                    'return': 1, 'error': 'Please specify the path to LLAMA2 dataset (pickle file)'}
+                    'return': 1, 'error': 'Llama2 70B validation dataset not present.'}
             if not os.path.exists(target_data_path):
-                cmds.append(f"mkdir {target_data_path}")
+                cmds.append(f"mkdir -p {target_data_path}")
             cmds.append(
-                f"ln -sf {env['MLC_NVIDIA_LLAMA_DATASET_FILE_PATH']} {target_data_file_path}")
+                f"ln -sf {env['MLC_DATASET_OPENORCA_PREPROCESSED_PATH']} {target_data_file_path}")
+
+        # check the presence of calibration dataset
+        if not env.get('LLAMA2_PRE_QUANTIZED_CHECKPOINT_PATH'):
+            if not os.path.exists(target_calibration_data_file_path):
+                if env.get('MLC_DATASET_OPENORCA_CALIBRATION_PATH', '') == '':
+                    return {
+                        'return': 1, 'error': 'Llama2 70B calibration dataset not present.'}
+                if not os.path.exists(target_data_path):
+                    cmds.append(f"mkdir -p {target_data_path}")
+                cmds.append(
+                    f"ln -sf {env['MLC_DATASET_OPENORCA_CALIBRATION_PATH']} {target_calibration_data_file_path}")
+
+        if not os.path.exists(preprocessed_data_for_accuracy_checker):
+            if not os.path.exists(preprocessed_data_for_accuracy_checker):
+                cmds.append(
+                    f"mkdir -p {os.path.dirname(preprocessed_data_for_accuracy_checker)}")
+            cmds.append(
+                f"ln -sf {env['MLC_DATASET_OPENORCA_PREPROCESSED_PATH']} {preprocessed_data_for_accuracy_checker}")
 
         model_name = "llama2-70b"
         model_path = fp8_model_path
@@ -550,6 +585,11 @@ def preprocess(i):
         if gpu_inference_streams:
             run_config += f" --gpu_inference_streams={gpu_inference_streams}"
 
+        model_precision = env.get(
+            'MLC_MLPERF_MODEL_PRECISION').replace('float', 'fp')
+        if model_precision:
+            run_config += f" --precision={model_precision}"
+
         dla_copy_streams = env.get(
             'MLC_MLPERF_NVIDIA_HARNESS_DLA_COPY_STREAMS')
         if dla_copy_streams:
@@ -688,8 +728,12 @@ def preprocess(i):
             run_config += f" --use_fp8"
 
         if "llama2" in env["MLC_MODEL"]:
-            run_config += f" --fp8_quant_model_path={fp8_model_path}"
-            run_config += f" --tensor_parallelism={tmp_tp_size}"
+            run_config += f" --checkpoint_dir={fp8_model_path}"
+            if env.get('MLC_MLPERF_INFERENCE_POST_5_0'):
+                run_config += f" --trtllm_build_flags=tensor_parallelism:{tmp_tp_size},pipeline_parallelism:{tmp_pp_size}"
+            else:
+                run_config += f" --tensor_parallelism={tmp_tp_size}"
+                run_config += f" --pipeline_parallelism={tmp_pp_size}"
 
         enable_sort = env.get('MLC_MLPERF_NVIDIA_HARNESS_ENABLE_SORT')
         if enable_sort and not is_false(enable_sort):
@@ -757,9 +801,11 @@ def preprocess(i):
         hpcx_paths.append("/opt/hpcx/ucx/lib")
     if os.path.exists("/opt/hpcx/ucc/lib"):
         hpcx_paths.append("/opt/hpcx/ucc/lib")
+    if os.path.exists("/opt/hpcx/ompi/lib"):
+        hpcx_paths.append("/opt/hpcx/ompi/lib")
 
     env['+LD_LIBRARY_PATH'] = hpcx_paths + env['+LD_LIBRARY_PATH']
-
+    env['+PYTHONPATH'] = []
     #    print(env)
 
     return {'return': 0}

script/app-mlperf-inference-nvidia/meta.yaml

@@ -355,6 +355,8 @@ variations:
     group: batchsize-format-change
   v5.0+:
     group: batchsize-format-change
+    env:
+      MLC_MLPERF_INFERENCE_POST_5_0: "yes"
   v5.0:
     base:
       - v5.0+
@@ -1279,13 +1281,17 @@ variations:
       MLC_MLPERF_NVIDIA_HARNESS_NUM_SORT_SEGMENTS: '2'
       MLC_MLPERF_NVIDIA_HARNESS_SKIP_POSTPROCESS: True
 
-  gpu_memory.80,pre5.0,num-gpus.2,llama2-70b,offline,run_harness:
+  gpu_memory.80,pre5.0,num-gpus.2,llama2-70b_,offline,run_harness:
     default_variations:
       batch-size: batch_size.896
 
-  gpu_memory.80,v5.0+,num-gpus.2,llama2-70b,offline,run_harness:
+  gpu_memory.80,v5.0+,num-gpus.2,llama2-70b_,offline,run_harness:
     default_variations:
       batch-size: batch_size."llama2-70b:1024"
+
+  gpu_memory.80,v5.0+,num-gpus.8,llama2-70b_,offline,run_harness:
+    default_variations:
+      batch-size: batch_size."llama2-70b:4096"
 
   gpu_memory.16,pre5.0,gptj_,offline,run_harness:
     default_variations:

script/app-mlperf-inference/meta.yaml

@@ -28,6 +28,8 @@ default_env:
   MLC_TEST_QUERY_COUNT: '10'
   MLC_MLPERF_QUANTIZATION: off
   MLC_GET_PLATFORM_DETAILS: no
+  MLC_NVIDIA_TP_SIZE: "2"
+  MLC_NVIDIA_PP_SIZE: "1"
 
 env:
   MLC_MLPERF_PRINT_SUMMARY: "no"
@@ -62,8 +64,8 @@ input_mapping:
   readme: MLC_MLPERF_README
   debug: MLC_DEBUG_SCRIPT_BENCHMARK_PROGRAM
   gpu_name: MLC_NVIDIA_GPU_NAME
-  nvidia_llama2_dataset_file_path: MLC_NVIDIA_LLAMA_DATASET_FILE_PATH
   tp_size: MLC_NVIDIA_TP_SIZE
+  pp_size: MLC_NVIDIA_PP_SIZE
   use_dataset_from_host: MLC_USE_DATASET_FROM_HOST
 
 predeps: False
@@ -324,9 +326,21 @@ variations:
             MLC_MLPERF_NVIDIA_SKIP_GPTJ:
               - "yes"
         - tags: get,ml-model,llama2-70b,_nvidia,_fp8
+          names:
+            - llama2-model  
           update_tags_from_env_with_prefix:
             _tp-size.:
               - MLC_NVIDIA_TP_SIZE
+            _pp-size.:
+              - MLC_NVIDIA_PP_SIZE
+          skip_if_env:
+            MLC_MLPERF_NVIDIA_SKIP_LLAMA2_70B:
+              - "yes"
+        - tags: get,dataset,preprocessed,openorca,_calibration,_mlcommons,_nvidia
+          skip_if_env:
+            MLC_MLPERF_NVIDIA_SKIP_LLAMA2_70B:
+              - "yes"
+        - tags: get,dataset,preprocessed,openorca,_validation,_mlcommons,_nvidia
           skip_if_env:
             MLC_MLPERF_NVIDIA_SKIP_LLAMA2_70B:
               - "yes"
@@ -505,29 +519,65 @@ variations:
       image_name: mlperf-inference-nvidia-v4.1-dev-llm
       deps:
         - tags: get,ml-model,llama2-70b,_nvidia,_fp8
+          names:
+            - llama2-model  
           update_tags_from_env_with_prefix:
             _tp-size.:
               - MLC_NVIDIA_TP_SIZE
+            _pp-size.:
+              - MLC_NVIDIA_PP_SIZE
+        - tags: get,dataset,preprocessed,openorca,_calibration,_mlcommons,_nvidia
+        - tags: get,dataset,preprocessed,openorca,_validation,_mlcommons,_nvidia
     env:
       BUILD_TRTLLM: 1
 
   nvidia-original,r4.1_default,llama2-70b_:
     docker:
       deps:
         - tags: get,ml-model,llama2-70b,_nvidia,_fp8
+          names:
+            - llama2-model  
           update_tags_from_env_with_prefix:
             _tp-size.:
               - MLC_NVIDIA_TP_SIZE
+            _pp-size.:
+              - MLC_NVIDIA_PP_SIZE
+        - tags: get,dataset,preprocessed,openorca,_calibration,_mlcommons,_nvidia
+        - tags: get,dataset,preprocessed,openorca,_validation,_mlcommons,_nvidia
     env:
       BUILD_TRTLLM: 1
 
   nvidia-original,r5.0_default,llama2-70b_:
     docker:
       deps:
         - tags: get,ml-model,llama2-70b,_nvidia,_fp8
+          names:
+            - llama2-model  
+          update_tags_from_env_with_prefix:
+            _tp-size.:
+              - MLC_NVIDIA_TP_SIZE
+            _pp-size.:
+              - MLC_NVIDIA_PP_SIZE
+        - tags: get,dataset,preprocessed,openorca,_calibration,_mlcommons,_nvidia
+        - tags: get,dataset,preprocessed,openorca,_validation,_mlcommons,_nvidia
+
+  nvidia-original,r5.1-dev_default,llama2-70b_:
+    default_variations:
+      precision: float8
+    docker:
+      deps:
+        - tags: get,ml-model,llama2-70b,_nvidia,_fp8,_v5.0
+          names:
+            - llama2-model  
           update_tags_from_env_with_prefix:
             _tp-size.:
               - MLC_NVIDIA_TP_SIZE
+            _pp-size.:
+              - MLC_NVIDIA_PP_SIZE
+        - tags: get,dataset,preprocessed,openorca,_calibration,_mlcommons,_nvidia
+        - tags: get,dataset,preprocessed,openorca,_validation,_mlcommons,_nvidia
+      env:
+        BUILD_TRTLLM: 1
 
   nvidia-original:
     docker:
@@ -594,6 +644,8 @@ variations:
         update_tags_from_env_with_prefix:
           "_gpu_memory." :
             - MLC_NVIDIA_GPU_MEMORY
+          "_num-gpus.":
+            - MLC_CUDA_NUM_DEVICES
         update_tags_from_env:
           - MLC_NVIDIA_HARNESS_GPU_VARIATION
 
@@ -1293,6 +1345,16 @@ variations:
             MLC_USE_MODEL_FROM_HOST:
             - 'yes'
           tags: get,ml-model,llama2
+          names:
+            - llama2-model  
+        - tags: get,dataset,preprocessed,openorca,_calibration,_mlcommons
+          enable_if_any_env:
+            MLC_USE_DATASET_FROM_HOST:
+            - 'yes'
+        - tags: get,dataset,preprocessed,openorca,_validation,_mlcommons
+          enable_if_any_env:
+            MLC_USE_DATASET_FROM_HOST:
+            - 'yes'
 
   llama2-70b_,amd:
     docker:
@@ -1306,6 +1368,8 @@ variations:
             MLC_USE_MODEL_FROM_HOST:
             - 'yes'
           tags: get,ml-model,llama2,_amd,_pytorch
+          names:
+            - llama2-model  
 
   mixtral-8x7b:
     group:
@@ -1830,6 +1894,12 @@ variations:
   fp32:
     alias: float32
 
+  fp4:
+    alias: float4
+
+  fp8:
+    alias: float8
+
   float32:
     group: precision
     default: true
@@ -1842,6 +1912,16 @@ variations:
       kilt-harness:
         tags: _fp32
 
+  float4:
+    group: precision
+    env:
+      MLC_MLPERF_MODEL_PRECISION: float4
+
+  float8:
+    group: precision
+    env:
+      MLC_MLPERF_MODEL_PRECISION: float8
+
   float16:
     group: precision
     env:
@@ -2128,10 +2208,10 @@ variations:
       reproducibility
     add_deps_recursive:
       nvidia-inference-common-code:
-        tags: _custom,_v5.1-dev
+        tags: _mlcommons,_v5.1-dev
       nvidia-inference-server:
         version: r5.0
-        tags: _custom
+        tags: _mlcommons
       nvidia-harness:   
         tags: _v5.0
       intel-harness:
@@ -2285,6 +2365,9 @@ docker:
    - "${{ GPTJ_CHECKPOINT_PATH }}:${{ GPTJ_CHECKPOINT_PATH }}"
    - "${{ MLC_CRITEO_PREPROCESSED_PATH }}:${{ MLC_CRITEO_PREPROCESSED_PATH }}"
    - "${{ LLAMA2_CHECKPOINT_PATH }}:${{ LLAMA2_CHECKPOINT_PATH }}"
+   - "${{ LLAMA2_PRE_QUANTIZED_CHECKPOINT_PATH }}:${{ LLAMA2_PRE_QUANTIZED_CHECKPOINT_PATH }}"
+   - "${{ MLC_DATASET_OPENORCA_PREPROCESSED_PATH }}:${{ MLC_DATASET_OPENORCA_PREPROCESSED_PATH }}"
+   - "${{ MLC_DATASET_OPENORCA_CALIBRATION_PATH }}:${{ MLC_DATASET_OPENORCA_CALIBRATION_PATH }}"
    - "${{ MLC_NVIDIA_LLAMA_DATASET_FILE_PATH }}:${{ MLC_NVIDIA_LLAMA_DATASET_FILE_PATH }}"
    - "${{ SDXL_CHECKPOINT_PATH }}:${{ SDXL_CHECKPOINT_PATH }}"
    - "${{ MLC_DATASET_KITS19_PREPROCESSED_PATH }}:${{ MLC_DATASET_KITS19_PREPROCESSED_PATH }}"
@@ -2314,3 +2397,4 @@ docker:
     intel_gptj_int8_model_path: MLC_MLPERF_INFERENCE_INTEL_GPTJ_INT8_MODEL_PATH
     nvidia_llama2_dataset_file_path: MLC_NVIDIA_LLAMA_DATASET_FILE_PATH
     tp_size: MLC_NVIDIA_TP_SIZE
+    pp_size: MLC_NVIDIA_PP_SIZE

script/build-mlperf-inference-server-nvidia/customize.py

@@ -1,6 +1,7 @@
 from mlc import utils
 import os
 import shutil
+from utils import *
 
 
 def preprocess(i):
@@ -18,6 +19,15 @@ def preprocess(i):
         env['+LIBRARY_PATH'].append(os.path.join(
             env['MLC_TENSORRT_INSTALL_PATH'], "lib"))
 
+    if is_true(env.get('BUILD_TRTLLM')):
+        hpcx_paths = []
+        if os.path.exists("/opt/hpcx/ucx/lib"):
+            hpcx_paths.append("/opt/hpcx/ucx/lib")
+        if os.path.exists("/opt/hpcx/ucc/lib"):
+            hpcx_paths.append("/opt/hpcx/ucc/lib")
+        if os.path.exists("/opt/hpcx/ompi/lib"):
+            hpcx_paths.append("/opt/hpcx/ompi/lib")
+
     cxxflags = [
         "-Wno-error=switch",
         "-DDALI_1_15=1",
@@ -38,6 +48,8 @@ def preprocess(i):
         env['+ CXXFLAGS'] = []
 
     env['+ CXXFLAGS'] += cxxflags
+    env['+LD_LIBRARY_PATH'] = hpcx_paths + env['+LD_LIBRARY_PATH']
+    env['+PYTHONPATH'] = []
     return {'return': 0}