Support llvm-install-from-src on Macbook, make gh actions code safe

.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml

@@ -32,8 +32,11 @@ jobs:
       run: |
         pip install mlcflow
     - name: Pull MLOps repository
+      env:
+        REPO: ${{ github.event.pull_request.head.repo.html_url }}
+        BRANCH: ${{ github.event.pull_request.head.ref }}  
       run: |
-        mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
+        mlc pull repo "$REPO" --branch="$BRANCH"
         mlcr --quiet --tags=get,sys-utils-mlc
     - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }}
       if: matrix.os == 'windows-latest'

.github/workflows/test-mlperf-inference-resnet50.yml

@@ -47,8 +47,11 @@ jobs:
         pip install tabulate
 
     - name: Pull MLOps repo
+      env:
+        REPO: ${{ github.event.pull_request.head.repo.html_url }}
+        BRANCH: ${{ github.event.pull_request.head.ref }}
       run: |
-        mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
+        mlc pull repo "$REPO" --branch="$BRANCH"
 
     - name: Test MLPerf Inference ResNet50 (Windows)
       if: matrix.os == 'windows-latest'

.github/workflows/test-mlperf-inference-retinanet.yml

@@ -45,8 +45,11 @@ jobs:
         pip install mlcflow
         pip install tabulate
     - name: Pull MLOps repo
+      env:
+        REPO: ${{ github.event.pull_request.head.repo.html_url }}
+        BRANCH: ${{ github.event.pull_request.head.ref }}
       run: |
-        mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
+        mlc pull repo "$REPO" --branch="$BRANCH"
 
     - name: Test MLPerf Inference Retinanet using ${{ matrix.backend }} on ${{ matrix.os }}
       if: matrix.os == 'windows-latest'

.github/workflows/test-mlperf-inference-submission-generation.yml

@@ -76,9 +76,12 @@ jobs:
         pip install mlcflow
         pip install tabulate
     - name: Pull MLOps repository
+      env:
+        REPO: ${{ github.event.pull_request.head.repo.html_url }}
+        BRANCH: ${{ github.event.pull_request.head.ref }}
       run: |
         if [ -z "${{ inputs.repo-url }}" ]; then
-          mlc pull repo ${{ github.event.pull_request.head.repo.html_url }} --branch=${{ github.event.pull_request.head.ref }}
+          mlc pull repo "$REPO" --branch="$BRANCH"
         else
           mlc pull repo mlcommons@mlperf-automations --branch=dev
         fi

.github/workflows/test-mlperf-inference-tvm-resnet50.yml

@@ -34,8 +34,8 @@ jobs:
         REPO: ${{ github.event.pull_request.head.repo.html_url }
         BRANCH: ${{ github.event.pull_request.head.ref }}
       run: |
-        mlc pull repo $REPO --branch=$BRANCH
-        mlcr --quiet --tags=get,sys-utils-cm
+        mlc pull repo "$REPO" --branch="$BRANCH"
+        mlcr --quiet --tags=get,sys-utils-mlc
     - name: Test MLC Tutorial TVM
       run: |
         mlcr run-mlperf,inference,_submission,_short --adr.python.name=mlperf --adr.python.version_min=3.8 --submitter=MLCommons --implementation=python --hw_name=gh_ubuntu-latest --model=resnet50 --backend=tvm-onnx --device=cpu --scenario=Offline --mode=accuracy --test_query_count=5 --clean --quiet ${{ matrix.extra-options }}

script/get-generic-sys-util/meta.yaml

@@ -67,6 +67,13 @@ variations:
         brew: autoconf
         dnf: autoconf
         yum: autoconf
+  coreutils:
+    env:
+      MLC_SYS_UTIL_NAME: coreutils
+      MLC_SYS_UTIL_CHECK_CMD: 'gtimeout 1s ls' 
+    state:
+      coreutils:
+        brew: coreutils
   cmake:
     env:
       MLC_SYS_UTIL_NAME: cmake
@@ -604,7 +611,7 @@ variations:
     state:
       ninja-build:
         apt: ninja-build
-        brew: ninja-build
+        brew: ninja
         dnf: ninja-build
         yum: ninja-build
         zypper: ninja-build

script/install-llvm-src/customize.py

@@ -1,5 +1,6 @@
 from mlc import utils
 from utils import is_true
+import platform
 import os
 
 
@@ -15,7 +16,7 @@ def preprocess(i):
     q = '"' if os_info['platform'] == 'windows' else "'"
 
     clang_file_name = "clang"
-    extra_cmake_options = ''
+    extra_cmake_options = env.get('MLC_LLVM_EXTRA_CMAKE_OPTIONS', '')
 
     if env.get('MLC_LLVM_INSTALLED_PATH', '') != '' and os.path.exists(
             env.get('MLC_LLVM_INSTALLED_PATH')):
@@ -50,10 +51,30 @@ def preprocess(i):
 
             llvm_build_type = env['MLC_LLVM_BUILD_TYPE']
 
-            targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86')
+            targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD')
+            host_platform = env.get('MLC_HOST_PLATFORM_FLAVOR')
+            if not targets_to_build:
+                if 'arm64' in host_platform:
+                    targets_to_build = 'AArch64'
+                else:
+                    targets_to_build = 'X86'
+
             cross_compile_options = env.get('MLC_LLVM_CROSS_COMPILE_FLAGS', '')
+            target_triple = env.get('MLC_LLVM_TARGET_TRIPLE', '')
+            compiler_rt_target_triple_string = ""
+
+            if target_triple != '':
+                target_triple_string = f""" -DLLVM_DEFAULT_TARGET_TRIPLE="{target_triple}" """
+            else:
+                if env.get('MLC_HOST_OS_TYPE',
+                           '') == 'darwin' and 'flang' in enable_projects:
+                    target_triple = get_target_triple()
+                    target_triple_string = f""" -DLLVM_DEFAULT_TARGET_TRIPLE="{target_triple}" """
+                    compiler_rt_target_triple_string = f""" -DCOMPILER_RT_DEFAULT_TARGET_TRIPLE="{target_triple}" """
+                else:
+                    target_triple_string = ""
 
-            cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type} -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON  -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {cross_compile_options} {extra_cmake_options}"""
+            cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type} -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON  -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {cross_compile_options} {target_triple_string} {compiler_rt_target_triple_string} {extra_cmake_options}"""
 
             env['MLC_LLVM_CMAKE_CMD'] = cmake_cmd
 
@@ -69,6 +90,22 @@ def preprocess(i):
     return {'return': 0}
 
 
+def get_target_triple():
+    machine = platform.machine()       # e.g. 'arm64' or 'x86_64'
+    system = platform.system().lower()  # e.g. 'darwin', 'linux'
+    release = platform.release()       # e.g. '24.6.0'
+
+    if system == "darwin":
+        # Darwin = macOS, append apple-darwin
+        return f"{machine}-apple-darwin{release}"
+    elif system == "linux":
+        return f"{machine}-pc-linux-gnu"
+    elif system == "windows":
+        return f"{machine}-pc-windows-msvc"
+    else:
+        return f"{machine}-{system}-{release}"
+
+
 def postprocess(i):
 
     env = i['env']

script/install-llvm-src/meta.yaml

@@ -38,7 +38,8 @@ deps:
     - MLC_VERSION
 input_mapping:
   targets: MLC_LLVM_TARGETS_TO_BUILD
-
+  target_triple: MLC_LLVM_TARGET_TRIPLE
+  extra_options: MLC_LLVM_EXTRA_CMAKE_OPTIONS
 env:
   MLC_GIT_URL: https://github.com/llvm/llvm-project
 name: Build LLVM compiler from sources (can take >30 min)