llvm-build.yml

name: LLVM Build

on:
  push:
    branches:
      - llvm-head
    paths:
      - cmake/llvm-hash.txt
  workflow_dispatch:

env:
  SCCACHE_DIR: ${{ github.workspace }}/sccache

permissions:
  contents: read
  id-token: write

jobs:

  build:
    name: Build on ${{ matrix.config.runner }}
    runs-on: ${{ matrix.config.runs_on }}
    timeout-minutes: 240  # 4 hours

    strategy:
      fail-fast: true
      matrix:
        config:
        - {runner: 'Ubuntu 20.04', runs_on: 'ubuntu-20.04', target-os: 'ubuntu', arch: 'x64'}
        - {runner: 'Ubuntu 20.04 ARM64', runs_on: 'ubuntu-20.04', target-os: 'ubuntu', arch: 'arm64'}
        - {runner: 'CentOS 7', runs_on: ['self-hosted', 'CPU'], target-os: 'centos', arch: 'x64'}
        - {runner: 'MacOS X64', runs_on: 'macos-12', target-os: 'macos', arch: 'x64'}
        - {runner: 'MacOS ARM64', runs_on: 'macos-12', target-os: 'macos', arch: 'arm64'}
        # TODO(#2805): add back once the workflow works and runs in comparable time to the other ones
        # - {runner: 'Windows Latest', runs_on: 'windows-latest', target-os: 'windows', arch: 'x64'}

    steps:

    - name: Checkout Repo
      uses: actions/checkout@v3
      with:
        path: llvm-build

    - name: Fetch LLVM Commit Hash
      shell: bash
      run: |
        LLVM_COMMIT_HASH="$(cat llvm-build/cmake/llvm-hash.txt)"
        echo "Found LLVM commit hash: ${LLVM_COMMIT_HASH}"
        echo "llvm_commit_hash=${LLVM_COMMIT_HASH}" >> ${GITHUB_ENV}

        SHORT_LLVM_COMMIT_HASH="${LLVM_COMMIT_HASH:0:8}"
        echo "Short LLVM commit hash: ${SHORT_LLVM_COMMIT_HASH}"
        echo "short_llvm_commit_hash=${SHORT_LLVM_COMMIT_HASH}" >> ${GITHUB_ENV}

        INSTALL_DIR="llvm-${SHORT_LLVM_COMMIT_HASH}-${{ matrix.config.target-os }}-${{ matrix.config.arch }}"
        echo "LLVM installation directory name: ${INSTALL_DIR}"
        echo "llvm_install_dir=${INSTALL_DIR}" >> ${GITHUB_ENV}

    - name: Checkout LLVM
      uses: actions/checkout@v3
      with:
        repository: llvm/llvm-project
        path: llvm-project
        ref: ${{ env.llvm_commit_hash }}

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: 3.11

    - name: Set up MSVC
      if: matrix.config.arch == 'x64' && (matrix.config.target-os == 'windows')
      uses: ilammy/msvc-dev-cmd@v1.4.1
      with:
        arch: amd64

    - name: Install Prerequisites
      shell: bash
      run: |
        python3 -m pip install cmake ninja sccache
        mkdir -p ${{ env.SCCACHE_DIR }}
        rm -rf ${{ env.SCCACHE_DIR }}/*

    - name: Enable Cache
      uses: actions/cache@v3
      with:
        path: ${{ env.SCCACHE_DIR }}
        key: ${{ matrix.config.target-os }}-${{ matrix.config.arch }}-${{ env.short_llvm_commit_hash }}
        restore-keys: ${{ matrix.config.target-os }}-${{ matrix.config.arch }}-

    - name: Configure, Build, Test, and Install LLVM (Ubuntu and macOS x64)
      if: matrix.config.arch == 'x64' && (matrix.config.target-os == 'ubuntu' || matrix.config.target-os == 'macos')
      run: >
        python3 -m pip install -r llvm-project/mlir/python/requirements.txt

        cmake -GNinja -Bllvm-project/build
        -DCMAKE_BUILD_TYPE=Release
        -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++
        -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
        -DCMAKE_INSTALL_PREFIX="${{ env.llvm_install_dir }}"
        -DCMAKE_LINKER=lld
        -DLLVM_BUILD_UTILS=ON
        -DLLVM_BUILD_TOOLS=ON
        -DLLVM_ENABLE_ASSERTIONS=ON
        -DMLIR_ENABLE_BINDINGS_PYTHON=ON
        -DLLVM_ENABLE_PROJECTS=mlir
        -DLLVM_INSTALL_UTILS=ON
        -DLLVM_TARGETS_TO_BUILD="host;NVPTX;AMDGPU"
        -DLLVM_ENABLE_TERMINFO=OFF
        llvm-project/llvm

        ninja -C llvm-project/build check-mlir install

        tar czf "${{ env.llvm_install_dir }}.tar.gz" "${{ env.llvm_install_dir }}"

    - name: Configure, Build, Test, and Install LLVM (Windows)
      if: matrix.config.arch == 'x64' && (matrix.config.target-os == 'windows')
      run: >
        python3 -m pip install -r llvm-project/mlir/python/requirements.txt

        cmake -GNinja -Bllvm-project/build
        -DCMAKE_BUILD_TYPE=Release
        -DCMAKE_C_COMPILER=cl -DCMAKE_CXX_COMPILER=cl
        -DCMAKE_INSTALL_PREFIX="${{ env.llvm_install_dir }}"
        -DLLVM_BUILD_UTILS=ON
        -DLLVM_BUILD_TOOLS=ON
        -DLLVM_ENABLE_ASSERTIONS=ON
        -DMLIR_ENABLE_BINDINGS_PYTHON=ON
        -DLLVM_ENABLE_PROJECTS="clang;mlir"
        -DLLVM_INSTALL_UTILS=ON
        -DLLVM_TARGETS_TO_BUILD="host;NVPTX;AMDGPU"
        -DLLVM_ENABLE_TERMINFO=OFF
        llvm-project/llvm

        ninja -C llvm-project/build check-mlir install

        tar czf "${{ env.llvm_install_dir }}.tar.gz" "${{ env.llvm_install_dir }}"


    - name: Configure, Build, and Install LLVM (ubuntu arm64)
      if: matrix.config.arch == 'arm64' && matrix.config.target-os == 'ubuntu'
      run: |
        python3 -m pip install -r llvm-project/mlir/python/requirements.txt
        mkdir arm-sysroot
        mkdir -p llvm-project/host-tools
        cd llvm-project/host-tools
        cmake -GNinja ../llvm -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="mlir;llvm;clang"
        ninja mlir-tblgen
        ninja llvm-tblgen
        ninja clang-tblgen
        cd ../..
        mv ./llvm-project/host-tools/bin ./host-tools
        HOST_TOOLS="$(pwd)/host-tools"
        rm -rf llvm-project/host-tools
        sudo apt-get update
        sudo apt-get install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf qemu-user-static gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
        cp -r /usr/aarch64-linux-gnu/lib ./arm-sysroot
        cp -r /usr/aarch64-linux-gnu/include ./arm-sysroot
        LINKER=$(pwd)/arm-sysroot/lib/ld-linux-aarch64.so.1
        wget http://ftp.de.debian.org/debian/pool/main/g/gcc-defaults/gcc-aarch64-linux-gnu_13.2.0-7_amd64.deb
        dpkg-deb -x gcc-aarch64-linux-gnu_13.2.0-7_amd64.deb ./arm-sysroot
        export LD_LIBRARY_PATH=$(pwd)/arm-sysroot/lib:$LD_LIBRARY_PATH
        sudo ln -s $LINKER /lib/ld-linux-aarch64.so.1
        SYSROOT="$(pwd)/arm-sysroot"
        echo $SYSROOT
        echo $LINKER
        cmake -GNinja -Bllvm-project/build \
        -DCMAKE_BUILD_TYPE=Release \
        -DLLVM_ENABLE_PROJECTS="mlir;llvm" \
        -DLLVM_BUILD_UTILS=ON \
        -DLLVM_TABLEGEN=$HOST_TOOLS/llvm-tblgen \
        -DMLIR_TABLEGEN=$HOST_TOOLS/mlir-tblgen \
        -DCLANG_TABLEGEN=$HOST_TOOLS/clang-tblgen \
        -DLLVM_ENABLE_ASSERTIONS=ON \
        -DCMAKE_LINKER=$LINKER \
        -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
        -DLLVM_ENABLE_ZSTD=OFF \
        -DLLVM_INSTALL_UTILS=ON \
        -DCMAKE_INSTALL_PREFIX="${{ env.llvm_install_dir }}" \
        -DLLVM_TARGETS_TO_BUILD="AArch64;NVPTX;AMDGPU" \
        -DCMAKE_CROSSCOMPILING=True \
        -DLLVM_TARGET_ARCH=AArch64 \
        -DLLVM_DEFAULT_TARGET_TRIPLE=aarch64-linux-gnu \
        -DLLVM_USE_HOST_TOOLS=OFF \
        -DCMAKE_C_COMPILER="/usr/bin/aarch64-linux-gnu-gcc" \
        -DCMAKE_CXX_COMPILER="/usr/bin/aarch64-linux-gnu-g++" \
        -DCMAKE_ASM_COMPILER="/usr/bin/aarch64-linux-gnu-as" \
        -DCMAKE_AR="/usr/bin/aarch64-linux-gnu-ar" \
        -DCMAKE_NM="/usr/bin/aarch64-linux-gnu-nm" \
        -DCMAKE_OBJCOPY="/usr/bin/aarch64-linux-gnu-objcopy" \
        -DCMAKE_OBJDUMP="/usr/bin/aarch64-linux-gnu-objdump" \
        -DCMAKE_RANLIB="/usr/bin/aarch64-linux-gnu-ranlib" \
        -DCMAKE_STRIP="/usr/bin/aarch64-linux-gnu-strip" \
        -DCMAKE_SYSROOT=$SYSROOT \
        -DLLVM_ENABLE_TERMINFO=OFF \
        llvm-project/llvm
        ninja -C llvm-project/build install
        CURR_PWD="$(pwd)"
        cd "${{ env.llvm_install_dir }}/python_packages/mlir_core/mlir/_mlir_libs/"
        for file in *x86_64*; do
          mv "$file" "${file/x86_64/aarch64}"
        done
        cd $CURR_PWD
        tar czf "${{ env.llvm_install_dir }}.tar.gz" "${{ env.llvm_install_dir }}"

    - name: Configure, Build, and Install LLVM (macOS arm64)
      if: matrix.config.arch == 'arm64' && matrix.config.target-os == 'macos'
      run: >
        python3 -m pip install -r llvm-project/mlir/python/requirements.txt

        cmake -GNinja -Bllvm-project/build
        -DCMAKE_BUILD_TYPE=Release
        -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++
        -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
        -DCMAKE_INSTALL_PREFIX="${{ env.llvm_install_dir }}"
        -DCMAKE_LINKER=lld
        -DCMAKE_OSX_ARCHITECTURES=arm64
        -DLLVM_BUILD_UTILS=ON
        -DLLVM_BUILD_TOOLS=ON
        -DLLVM_ENABLE_ASSERTIONS=ON
        -DMLIR_ENABLE_BINDINGS_PYTHON=ON
        -DLLVM_ENABLE_PROJECTS=mlir
        -DLLVM_ENABLE_ZSTD=OFF
        -DLLVM_INSTALL_UTILS=ON
        -DLLVM_TARGETS_TO_BUILD="AArch64;NVPTX;AMDGPU"
        -DLLVM_USE_HOST_TOOLS=ON
        -DLLVM_ENABLE_TERMINFO=OFF
        llvm-project/llvm

        ninja -C llvm-project/build install

        tar czf "${{ env.llvm_install_dir }}.tar.gz" "${{ env.llvm_install_dir }}"


    - name: Configure, Build, Test, and Install LLVM (CentOS)
      if: matrix.config.target-os == 'centos'
      run: |
        # if this step crashes, it can leave behind a stale docker container
        docker container prune -f
        docker rmi -f $(docker images -q)

        docker build --tag llvm-build --build-arg llvm_dir=llvm-project \
          -f llvm-build/.github/workflows/llvm-build/Dockerfile .

        # Create temporary container to copy cache and installed artifacts.
        CONTAINER_ID=$(docker create llvm-build)
        docker cp "${CONTAINER_ID}:/install" "${{ env.llvm_install_dir }}"
        tar czf "${{ env.llvm_install_dir }}.tar.gz" "${{ env.llvm_install_dir }}"

        # We remove the existing directory, otherwise docker will
        # create a subdirectory inside the existing directory.
        rm -rf "${{ env.SCCACHE_DIR }}"
        docker cp "${CONTAINER_ID}:/sccache" "${{ env.SCCACHE_DIR }}"
        sudo chown -R "$(id -u -n):$(id -g -n)" "${{ env.SCCACHE_DIR }}"

        docker rm "${CONTAINER_ID}"

    - name: Upload Build Artifacts
      uses: actions/upload-artifact@v3
      with:
        name: llvm-${{ matrix.config.target-os }}-${{ matrix.config.arch }}
        path: |
          ${{ github.workspace }}/llvm-*.tar.gz

    - name: Azure Login
      if: ${{ (github.repository == 'openai/triton') }}
      uses: azure/login@v1
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

    - name: Upload LLVM Artifacts to Azure
      if: ${{ (github.repository == 'openai/triton') }}
      run: |
        az storage blob upload --account-name tritonlang --auth-mode login --container-name llvm-builds --file "${{ env.llvm_install_dir }}.tar.gz" --name "${{ env.llvm_install_dir }}.tar.gz" --overwrite

        URL=$(az storage blob url --account-name tritonlang --auth-mode login --container-name llvm-builds --name "${{ env.llvm_install_dir }}.tar.gz")
        echo "Blob URL: ${URL}"

    - name: Azure Logout
      if: ${{ (github.repository == 'openai/triton') }}
      run: |
        az logout
        az cache purge
        az account clear

    - name: Dump Sccache Statistics
      run: sccache --show-stats