Support env in os_info, cross-compilation for llvm

automation/utils.py

@@ -14,12 +14,12 @@ def get_host_os_info(i={}):
     Get some host platform name (currently windows or linux) and OS bits
 
     Args:
-       (CM input dict):
+       (input dict):
 
        (bits) (str): force host platform bits
 
     Returns:
-       (CM return dict):
+       (dict):
 
        * return (int): return code == 0 if no error and >0 if error
        * (error) (str): error string if return>0
@@ -41,7 +41,7 @@ def get_host_os_info(i={}):
     pbits = str(8 * struct.calcsize("P"))
 
     if platform.system().lower().startswith('win'):
-        platform = 'windows'
+        platform_name = 'windows'
         info['bat_ext'] = '.bat'
         info['set_env'] = 'set ${key}=${value}'
         info['env_separator'] = ';'
@@ -56,9 +56,9 @@ def get_host_os_info(i={}):
         }
     else:
         if platform.system().lower().startswith('darwin'):
-            platform = 'darwin'
+            platform_name = 'darwin'
         else:
-            platform = 'linux'
+            platform_name = 'linux'
 
         info['bat_ext'] = '.sh'
         info['set_env'] = 'export ${key}="${value}"'
@@ -72,12 +72,14 @@ def get_host_os_info(i={}):
         info['start_script'] = ['#!/bin/bash', '']
         info['env'] = {}
 
-    info['platform'] = platform
+    info['platform'] = platform_name
+    info['env']['MLC_HOST_PLATFORM_FLAVOR'] = platform.machine()
+    info['env']['MLC_HOST_OS_TYPE'] = platform_name
 
     obits = i.get('bits', '')
     if obits == '':
         obits = '32'
-        if platform == 'windows':
+        if platform_name == 'windows':
             # Trying to get fast way to detect bits
             if os.environ.get('ProgramW6432', '') != '' or os.environ.get(
                     'ProgramFiles(x86)', '') != '':  # pragma: no cover

script/get-generic-sys-util/meta.yaml

@@ -628,6 +628,13 @@ variations:
         brew: ''
         dnf: ntpdate
         yum: ntpdate
+  crossbuild-essential-arm64:
+    env:
+      MLC_SYS_UTIL_NAME: crossbuild-essential-arm64
+      MLC_SYS_UTIL_CHECK_CMD: 'command -v aarch64-linux-gnu-gcc >/dev/null 2>&1'
+    state:
+      crossbuild-essential-arm64:
+        apt: crossbuild-essential-arm64
   numactl:
     deps:
     - enable_if_env:

script/install-gcc-src/customize.py

@@ -22,6 +22,11 @@ def preprocess(i):
     else:
         env['MLC_GCC_TARGET_STRING'] = ''
 
+    if env.get('MLC_GCC_HOST', '') != '':
+        env['MLC_GCC_HOST_STRING'] = f""" --target={env['MLC_GCC_HOST']} """
+    else:
+        env['MLC_GCC_HOST_STRING'] = ''
+
     env['MLC_GCC_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install', 'bin')
 
     return {'return': 0}

script/install-gcc-src/meta.yaml

@@ -26,6 +26,7 @@ deps:
     - MLC_GIT_CHECKOUT_TAG
 input_mapping:
   targets: MLC_GCC_TARGET 
+  host: MLC_GCC_HOST
 
 env:
   MLC_GIT_URL: git://gcc.gnu.org/git/gcc.git

script/install-gcc-src/run.sh

@@ -23,7 +23,7 @@ cd src
 cd ../build
 
 
-cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} --with-gcc-major-version-only --disable-multilib"
+cmd="../src/configure --prefix="${INSTALL_DIR}" ${MLC_GCC_TARGET_STRING} ${MLC_GCC_HOST_STRING} --with-gcc-major-version-only --disable-multilib"
 echo $cmd
 eval $cmd
 

script/install-llvm-src/customize.py

@@ -51,8 +51,9 @@ def preprocess(i):
             llvm_build_type = env['MLC_LLVM_BUILD_TYPE']
 
             targets_to_build = env.get('MLC_LLVM_TARGETS_TO_BUILD', 'X86')
+            cross_compile_options = env.get('MLC_LLVM_CROSS_COMPILE_FLAGS', '')
 
-            cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type} -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON  -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {extra_cmake_options}"""
+            cmake_cmd = f"""cmake {os.path.join(env["MLC_LLVM_SRC_REPO_PATH"], "llvm")} -GNinja -DCMAKE_BUILD_TYPE={llvm_build_type} -DLLVM_ENABLE_PROJECTS={q}{enable_projects}{q} -DLLVM_ENABLE_RUNTIMES={q}{enable_runtimes}{q} -DCMAKE_INSTALL_PREFIX={q}{install_prefix}{q} -DLLVM_ENABLE_RTTI=ON  -DLLVM_INSTALL_UTILS=ON -DLLVM_TARGETS_TO_BUILD={targets_to_build} {cross_compile_options} {extra_cmake_options}"""
 
             env['MLC_LLVM_CMAKE_CMD'] = cmake_cmd
 

script/install-llvm-src/meta.yaml

@@ -110,6 +110,11 @@ variations:
     env:
        MLC_LLVM_INSTALLED_PATH: '#'
        MLC_LLVM_USE_INSTALLED_DIR: yes
+  cross-compile-x86-aarch64:
+    deps:
+      - tags: get,generic-sys-util,_crossbuild-essential-arm64
+    env:
+      MLC_LLVM_CROSS_COMPILE_FLAGS: "-DCMAKE_SYSTEM_NAME=Linux  -DCMAKE_SYSTEM_PROCESSOR=aarch64 -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-linux-gnu -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc  -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++"
   for-intel-mlperf-inference-v3.1-bert:
     adr:
       conda-package: