various updates (#1124)

cm-mlops/automation/script/module.py

@@ -1940,7 +1940,7 @@ def _get_variation_tags_from_default_variations(self, variation_meta, variations
                     if tag_to_append not in variations:
                         variation_tag_static = self._get_name_for_dynamic_variation_tag(tag_to_append)
                         if not variation_tag_static or variation_tag_static not in variations:
-                            return {'return': 1, 'error': 'Invalid variation "{}" specified in default variations for the variation "{}" '.format(tag_to_append, variation_name)}
+                            return {'return': 1, 'error': 'Invalid variation "{}" specified in default variations for the variation "{}" '.format(tag_to_append, variation_meta)}
                     tmp_variation_tags.append(tag_to_append)
 
         return {'return': 0, 'variations_to_add': tmp_variation_tags}

cm-mlops/script/app-mlperf-inference-reference/_cm.yaml

@@ -563,7 +563,7 @@ prehook_deps:
     - remote-run-cmds
     tags: remote,run,cmds
     enable_if_env:
-      CM_SSH_RUN_COMMANDS:
+      CM_ASSH_RUN_COMMANDS:
       - "on"
 
 posthook_deps: 

cm-mlops/script/app-mlperf-inference/_cm.yaml

@@ -328,7 +328,7 @@ variations:
 
   kilt,qualcomm,bert-99.9:
     default_variations:
-      precision: fp16
+      precision: float16
 
   resnet50:
     group:

cm-mlops/script/benchmark-any-mlperf-inference-implementation/_cm.yaml

@@ -291,3 +291,27 @@ variations:
           glow:
             offline_target_qps: 350
             server_target_qps: 300
+  rb6:
+    group: sut
+    default_env:
+      CATEGORY: edge
+      DIVISION: closed
+    default_variations:
+      implementation: qualcomm
+
+  rb6,qualcomm:
+    default_env:
+      EXTRA_ARGS: " --adr.mlperf-inference-implementation.tags=_rb6 --env.CM_MLPERF_SHORT_RANGING_RUN=no"
+    state:
+      resnet50:
+        qaic:
+          glow:
+            offline_target_qps: 6800
+      retinanet:
+        qaic:
+          glow:
+            offline_target_qps: 125
+      bert-99:
+        qaic:
+          glow:
+            offline_target_qps: 255

cm-mlops/script/calibrate-model-for.qaic/_cm.json

@@ -74,7 +74,6 @@
   "variations": {
     "bs.1": {
       "group": "batch-size",
-      "default": true,
       "env": {
         "CM_QAIC_MODEL_BATCH_SIZE": "1",
         "CM_CREATE_INPUT_BATCH": "yes"

cm-mlops/script/compile-model-for.qaic/_cm.json

@@ -5,7 +5,7 @@
   "cache": true,
   "category": "AI/ML optimization",
   "new_env_keys": [
-    "CM_QAIC_*",
+    "CM_QAIC_MODEL*",
     "CM_ML_MODEL_FILE_WITH_PATH"
   ],
   "input_mapping": {
@@ -204,7 +204,8 @@
     },
     "resnet50,multistream": {
       "env": {
-        "CM_QAIC_MODEL_COMPILER_ARGS": ""
+        "CM_QAIC_MODEL_COMPILER_ARGS": "",
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=4 -mos=1 -ols=1"
       }
     },
     "resnet50,server": {
@@ -214,13 +215,13 @@
     "bert-99": {
       "adr": {
         "model-src": {
-          "tags": "bert-large,_onnx"
+          "tags": "bert-large,_onnx,_packed"
         }
       },
       "env": {
         "CM_COMPILE_BERT": "on",
         "CM_QAIC_MODEL_TO_CONVERT": "calibrate_bert_mlperf",
-        "CM_QAIC_MODEL_COMPILER_PARAMS_BASE": "-aic-hw -aic-hw-version=2.0 -execute-nodes-in-fp16=Mul,Sqrt,Div,Add,ReduceMean,Softmax,Sub,Gather,Erf,Pow,Concat,Tile,LayerNormalization -quantization-schema=symmetric_with_uint8 -quantization-precision=Int8 -quantization-precision-bias=Int32 -vvv -compile-only -onnx-define-symbol=batch_size,1 -onnx-define-symbol=seg_length,384 -multicast-weights -combine-inputs=false -combine-outputs=false",
+        "CM_QAIC_MODEL_COMPILER_PARAMS_BASE": "-aic-hw -aic-hw-version=2.0 -execute-nodes-in-fp16=Add,Div,Erf,Softmax -quantization-schema=symmetric_with_uint8 -quantization-precision=Int8 -quantization-precision-bias=Int32 -vvv -compile-only -onnx-define-symbol=batch_size,1 -onnx-define-symbol=seg_length,384 -multicast-weights -combine-inputs=false -combine-outputs=false",
         "CM_QAIC_MODEL_COMPILER_ARGS": ""
       },
       "deps": [
@@ -236,7 +237,12 @@
     "bert-99,offline": {
       "env": {
         "CM_QAIC_MODEL_COMPILER_ARGS": "-allocator-dealloc-delay=2 -size-split-granularity=1536 -vtcm-working-set-limit-ratio=1",
-        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=1 -mos=1 -ols=3"
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=1 -mos=1 -ols=2"
+      }
+    },
+    "bert-99,offline,nsp.16": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=1 -mos=1 -ols=2"
       }
     },
     "bert-99,offline,nsp.14": {
@@ -266,6 +272,47 @@
         "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=8 -mos=8 -ols=1"
       }
     },
+    "bert-99.9": {
+      "base": [
+        "no-quantized"
+      ],
+      "adr": {
+        "model-src": {
+          "tags": "bert-large,_onnx,_packed"
+        }
+      },
+      "env": {
+        "CM_COMPILE_BERT": "on",
+        "CM_QAIC_MODEL_TO_CONVERT": "bert_mlperf",
+        "CM_QAIC_MODEL_COMPILER_PARAMS_BASE": "-aic-hw -aic-hw-version=2.0 -convert-to-fp16 -vvv -compile-only -onnx-define-symbol=batch_size,1 -onnx-define-symbol=seg_length,384 -combine-inputs=false -combine-outputs=false",
+        "CM_QAIC_MODEL_COMPILER_ARGS": ""
+      }
+    },
+    "bert-99.9,offline": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=2 -mos=1 -ols=2"
+      }
+    },
+    "bert-99.9,offline,nsp.16": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=2 -mos=1 -ols=2"
+      }
+    },
+    "bert-99.9,offline,nsp.14": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=2 -mos=1 -ols=2"
+      }
+    },
+    "bert-99.9,server": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=4"
+      }
+    },
+    "bert-99.9,server,nsp.14": {
+      "env": {
+        "CM_QAIC_MODEL_COMPILER_ARGS_SUT": "-aic-num-cores=4"
+      }
+    },
     "tf": {
       "group": "model-framework"
     },

cm-mlops/script/compile-model-for.qaic/customize.py

@@ -22,6 +22,8 @@ def preprocess(i):
 
         print("Compiling from "+ os.getcwd())
 
+        env['CM_QAIC_MODEL_FINAL_COMPILATION_CMD'] = cmd
+
         env['CM_RUN_CMD'] = cmd
     else:
         import shutil

cm-mlops/script/compile-model-for.qaic/run.sh

@@ -24,5 +24,5 @@ function run() {
 }
 
 #Add your run commands here...
-#run "rm -rf elfs" 
+run "rm -rf elfs" 
 run "$CM_RUN_CMD"

cm-mlops/script/compile-program/customize.py

@@ -16,7 +16,8 @@ def preprocess(i):
     env['CM_F_INCLUDE_PATH'] = " -I".join([" "] + env.get('+F_INCLUDE_PATH', []) + CPATH)
 
     # If windows, need to extend it more ...
-    if os_info['platform'] == 'windows':
+    if os_info['platform'] == 'windows' and env.get('CM_COMPILER_FAMILY','')!='LLVM':
+        print ("WARNING: compile-program script should be extended to support flags for non-LLVM compilers on Windows")
         return {'return':0}
 
     LDFLAGS = env.get('+ LDFLAGS', [])

cm-mlops/script/compile-program/run.bat

@@ -22,14 +22,14 @@ cd %CM_SOURCE_FOLDER_PATH%
 IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
 
 if not "%CM_C_SOURCE_FILES%" == "" (
-  echo %CM_C_COMPILER_WITH_PATH% %CM_C_COMPILER_FLAGS% %CM_C_INCLUDE_PATH% %CM_C_SOURCE_FILES% %CM_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
-  "%CM_C_COMPILER_WITH_PATH%" %CM_C_COMPILER_FLAGS% %CM_C_INCLUDE_PATH% %CM_C_SOURCE_FILES% %CM_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
+  echo %CM_C_COMPILER_WITH_PATH% %CM_C_COMPILER_FLAGS% %CM_C_INCLUDE_PATH% %CM_C_SOURCE_FILES% %CM_LD_LIBRARY_PATH% %LDCFLAGS% %CM_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
+  "%CM_C_COMPILER_WITH_PATH%" %CM_C_COMPILER_FLAGS% %CM_C_INCLUDE_PATH% %CM_C_SOURCE_FILES% %CM_LD_LIBRARY_PATH% %LDCFLAGS% %CM_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
   IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
 )
 
 if not "%CM_CXX_SOURCE_FILES%" == "" (
-  echo %CM_CXX_COMPILER_WITH_PATH% %CM_CXX_COMPILER_FLAGS% %CM_CXX_INCLUDE_PATH% %CM_CXX_SOURCE_FILES% %CM_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
-  "%CM_CXX_COMPILER_WITH_PATH%" %CM_CXX_COMPILER_FLAGS% %CM_CXX_INCLUDE_PATH% %CM_CXX_SOURCE_FILES% %CM_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
+  echo %CM_CXX_COMPILER_WITH_PATH%  %CM_CXX_SOURCE_FILES% %CM_CXX_COMPILER_FLAGS% %CM_CPLUS_INCLUDE_PATH% %CM_LD_LIBRARY_PATH% %LDCXXFLAGS% %CM_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
+  "%CM_CXX_COMPILER_WITH_PATH%"  %CM_CXX_SOURCE_FILES% %CM_CXX_COMPILER_FLAGS% %CM_CPLUS_INCLUDE_PATH% %CM_LD_LIBRARY_PATH% %LDCXXFLAGS% %CM_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%"
   IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
 )
 

cm-mlops/script/get-compiler-flags/customize.py

@@ -4,7 +4,7 @@
 
 def preprocess(i):
     os_info = i['os_info']
-    
+
     env = i['env']
     env['+ CFLAGS'] = []
     env['+ CXXFLAGS'] = []
@@ -17,14 +17,14 @@ def preprocess(i):
 
     if env.get("CM_FAST_COMPILATION") in [ "yes", "on", "1" ]:
         DEFAULT_COMPILER_FLAGS = env.get("CM_COMPILER_FLAGS_FAST", "-O3")
-        DEFAULT_LINKER_FLAGS = env.get("CM_LINKER_FLAGS_FAST", "-O3 -flto")
+        DEFAULT_LINKER_FLAGS = env.get("CM_LINKER_FLAGS_FAST", "-O3") # -flto") - this flag is not always available
     elif env.get("CM_DEBUG_COMPILATION") in ["yes", "on", "1" ]:
         DEFAULT_COMPILER_FLAGS = env.get("CM_COMPILER_FLAGS_DEBUG", "-O0")
         DEFAULT_LINKER_FLAGS = env.get("CM_LINKER_FLAGS_DEBUG", "-O0")
     else:
         DEFAULT_COMPILER_FLAGS = env.get("CM_COMPILER_FLAGS_DEFAULT", "-O2")
         DEFAULT_LINKER_FLAGS = env.get("CM_LINKER_FLAGS_DEFAULT", "-O2")
-    
+
     env['+ CFLAGS'] += DEFAULT_COMPILER_FLAGS.split(" ")
     env['+ CXXFLAGS'] += DEFAULT_COMPILER_FLAGS.split(" ")
     env['+ FFLAGS'] += DEFAULT_COMPILER_FLAGS.split(" ")

cm-mlops/script/get-cuda/customize.py

@@ -206,7 +206,9 @@ def postprocess(i):
     if '+ LDFLAGS' not in env:
         env['+ LDFLAGS'] = []
     if 'CM_CUDA_PATH_LIB' in env and not cuda_system_path_install:
-        env['+ LDFLAGS'].append("-L"+env['CM_CUDA_PATH_LIB'])
+        x = env['CM_CUDA_PATH_LIB']
+        if ' ' in x: x='"'+x+'"'
+        env['+ LDFLAGS'].append("-L"+x)
 
     env['CM_CUDA_VERSION_STRING'] = "cu"+env['CM_CUDA_VERSION'].replace(".", "")
 

cm-mlops/script/get-llvm/customize.py

@@ -71,14 +71,16 @@ def postprocess(i):
     env['CM_C_COMPILER_WITH_PATH']=found_file_path
     env['CM_C_COMPILER_FLAG_OUTPUT']='-o '
     env['CM_C_COMPILER_FLAG_VERSION']='--version'
+    env['CM_C_COMPILER_FLAG_INCLUDE']='-I'
 
     env['CM_CXX_COMPILER_BIN']=file_name_cpp
     env['CM_CXX_COMPILER_WITH_PATH']=os.path.join(found_path, file_name_cpp)
     env['CM_CXX_COMPILER_FLAG_OUTPUT']='-o '
     env['CM_CXX_COMPILER_FLAG_VERSION']='--version'
+    env['CM_CXX_COMPILER_FLAG_INCLUDE']='-I'
 
-    env['CM_COMPILER_FLAGS_FAST'] = "-O3"
-    env['CM_LINKER_FLAGS_FAST'] = "-O3 -flto"
+    env['CM_COMPILER_FLAGS_FAST'] = "-O4"
+    env['CM_LINKER_FLAGS_FAST'] = "-O4" # "-flto" - this flag is not always available (requires LLVMgold.so)
     env['CM_COMPILER_FLAGS_DEBUG'] = "-O0"
     env['CM_LINKER_FLAGS_DEBUG'] = "-O0"
     env['CM_COMPILER_FLAGS_DEFAULT'] = "-O2"

cm-mlops/script/get-mlperf-inference-loadgen/_cm.json

@@ -69,7 +69,7 @@
         "cmake"
       ],
       "tags": "get,cmake",
-      "version_min": "3.1"
+      "version_min": "3.12"
     },
     {
       "names": [

cm-mlops/script/get-onnxruntime-prebuilt/_cm.json

@@ -12,6 +12,9 @@
     }
   ],
   "new_env_keys": [
+    "CM_ONNXRUNTIME_LIB_PATH",
+    "CM_ONNXRUNTIME_INCLUDE_PATH",
+    "+PATH",
     "+C_INCLUDE_PATH",
     "+CPLUS_INCLUDE_PATH",
     "+LD_LIBRARY_PATH",

cm-mlops/script/get-onnxruntime-prebuilt/customize.py

@@ -45,19 +45,30 @@ def postprocess(i):
 
     env = i['env']
 
+    hostos=env['CM_HOST_OS_TYPE']
+
     install_folder = env['CM_TMP_INSTALL_FOLDER']
 
     for key in ['+C_INCLUDE_PATH', '+CPLUS_INCLUDE_PATH', '+LD_LIBRARY_PATH', '+DYLD_FALLBACK_LIBRARY_PATH']:
 #        20221024: we save and restore env in the main script and can clean env here for determinism
 #        if key not in env:
         env[key] = []
 
-    env['+C_INCLUDE_PATH'].append(os.path.join(os.getcwd(), 'install', install_folder, 'include'))
-    env['+CPLUS_INCLUDE_PATH'].append(os.path.join(os.getcwd(), 'install', install_folder, 'include'))
+    include_path = os.path.join(os.getcwd(), 'install', install_folder, 'include')
+
+    env['+C_INCLUDE_PATH'].append(include_path)
+    env['+CPLUS_INCLUDE_PATH'].append(include_path)
 
     lib_path = os.path.join(os.getcwd(), 'install', install_folder, 'lib')
 
     env['+LD_LIBRARY_PATH'].append(lib_path)
     env['+DYLD_FALLBACK_LIBRARY_PATH'].append(lib_path)
 
+    if hostos =='windows': 
+        # For dynamic libraries
+        env['+PATH'] = [lib_path]
+
+    env['CM_ONNXRUNTIME_LIB_PATH'] = lib_path
+    env['CM_ONNXRUNTIME_INCLUDE_PATH'] = include_path
+
     return {'return':0}