Separate workflows

.github/workflows/build-wheel-linux-arm64-python310.yaml

@@ -0,0 +1,300 @@
+name: Build Catalyst Wheel on Linux (arm64)
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - labeled
+  push:
+    branches: [ main ]
+  workflow_dispatch:
+
+env:
+  MACOSX_DEPLOYMENT_TARGET: 14.0
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check_if_wheel_build_required:
+    uses: ./.github/workflows/check-for-wheel-build.yml
+
+  constants:
+    needs: [check_if_wheel_build_required]
+    if: needs.check_if_wheel_build_required.outputs.build-wheels == 'true'
+    name: "Set build matrix"
+    uses: ./.github/workflows/constants.yaml
+
+  build-dependencies:
+    needs: [constants, check_if_wheel_build_required]
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: [{official: "3.9",  subversion: "19", package: "python39"}]
+        container_img: ["quay.io/pypa/manylinux_2_28_aarch64"]
+        container_name: ["manylinux_2_28_aarch64"]
+
+    name: Build Dependencies (Python ${{ matrix.python_version.official }})
+    runs-on:
+      group: 'Office 24th floor M2 Mac'
+
+    if: needs.check_if_wheel_build_required.outputs.build-wheels == 'true'
+
+    steps:
+    - name: Checkout Catalyst repo
+      uses: actions/checkout@v4
+
+    - name: Setup Runner Environment
+      id: setup_env
+      uses: ./.github/workflows/utils/setup_self_hosted_macos_env_linux_arm64
+      with:
+        python_version: ${{ matrix.python_version.official }}
+
+    # Cache external project sources
+    - name: Cache LLVM Source
+      id: cache-llvm-source
+      uses: actions/cache@v4
+      with:
+        path:  mlir/llvm-project
+        key: ${{ runner.os }}-${{ matrix.container_name }}-llvm-${{ needs.constants.outputs.llvm_version }}-generic-source
+        enableCrossOsArchive: True
+
+    - name: Cache MHLO Source
+      id: cache-mhlo-source
+      uses: actions/cache@v4
+      with:
+        path: mlir/mlir-hlo
+        key: ${{ runner.os }}-${{ matrix.container_name }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-source
+        enableCrossOsArchive: True
+
+    - name: Cache Enzyme Source
+      id: cache-enzyme-source
+      uses: actions/cache@v4
+      with:
+        path:  mlir/Enzyme
+        key: ${{ runner.os }}-${{ matrix.container_name }}-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source
+        enableCrossOsArchive: True
+
+    - name: Clone LLVM Submodule
+      if: steps.cache-llvm-source.outputs.cache-hit != 'true'
+      uses: actions/checkout@v4
+      with:
+        repository: llvm/llvm-project
+        ref: ${{ needs.constants.outputs.llvm_version }}
+        path: mlir/llvm-project
+
+    - name: Clone MHLO Submodule
+      if: steps.cache-mhlo-source.outputs.cache-hit != 'true'
+      uses: actions/checkout@v4
+      with:
+        repository: tensorflow/mlir-hlo
+        ref: ${{ needs.constants.outputs.mhlo_version }}
+        path: mlir/mlir-hlo
+
+    - name: Clone Enzyme Submodule
+      if: steps.cache-enzyme-source.outputs.cache-hit != 'true'
+      uses: actions/checkout@v4
+      with:
+        repository: EnzymeAD/Enzyme
+        ref: ${{ needs.constants.outputs.enzyme_version }}
+        path: mlir/Enzyme
+
+    # Cache external project builds
+    - name: Cache LLVM Build
+      id: cache-llvm-build
+      uses: actions/cache@v4
+      with:
+        path:  llvm-build
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.container_name }}-llvm-${{ needs.constants.outputs.llvm_version }}-generic-build
+
+    - name: Cache MHLO Build
+      id: cache-mhlo-build
+      uses: actions/cache@v4
+      with:
+        path:  mhlo-build
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.container_name }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-build
+
+    - name: Cache Enzyme Build
+      id: cache-enzyme-build
+      uses: actions/cache@v4
+      with:
+        path: enzyme-build
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.container_name }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-generic-build
+
+    - name: Build LLVM / MLIR
+      if: steps.cache-llvm-build.outputs.cache-hit != 'true'
+      run: |
+        set -x
+        # With GCC 13, LLVM fails some tests, then we use GCC 12 instead
+        export GCC_VERSION=12
+        docker run --rm --platform linux/aarch64 \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            -v `pwd`:/catalyst \
+            -i ${{ matrix.container_img }} \
+            bash /catalyst/.github/workflows/scripts/linux_arm64/rh8/build_llvm.sh $GCC_VERSION ${{ matrix.python_version.official }} ${{ matrix.python_version.subversion }} ${{ matrix.python_version.package }}
+
+    - name: Build MHLO Dialect
+      if: steps.cache-mhlo-build.outputs.cache-hit != 'true'
+      run: |
+        set -x
+        export GCC_VERSION=13
+        docker run --rm --platform linux/aarch64 \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            -v `pwd`:/catalyst \
+            -i ${{ matrix.container_img }} \
+            bash /catalyst/.github/workflows/scripts/linux_arm64/rh8/build_mhlo.sh $GCC_VERSION ${{ matrix.python_version.official }} ${{ matrix.python_version.subversion }} ${{ matrix.python_version.package }}
+
+    - name: Build Enzyme
+      if: steps.cache-enzyme-build.outputs.cache-hit != 'true'
+      run: |
+        set -x
+        export GCC_VERSION=13
+        docker run --rm --platform linux/aarch64 \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            -v `pwd`:/catalyst \
+            -i ${{ matrix.container_img }} \
+            bash /catalyst/.github/workflows/scripts/linux_arm64/rh8/build_enzyme.sh $GCC_VERSION ${{ matrix.python_version.official }} ${{ matrix.python_version.subversion }} ${{ matrix.python_version.package }}
+
+  catalyst-linux-wheels-arm64:
+    needs: [constants, build-dependencies]
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: [{official: "3.10", subversion: "14", package: "python3.10", alternative: "310"}]
+        container_img: ["quay.io/pypa/manylinux_2_28_aarch64"]
+        container_name: ["manylinux_2_28_aarch64"]
+
+    name: Build Wheels (Python ${{ matrix.python_version.official }})
+    runs-on:
+      group: 'Office 24th floor M2 Mac'
+
+    steps:
+    - name: Checkout Catalyst repo
+      uses: actions/checkout@v4
+
+    - name: Setup Runner Environment
+      id: setup_env
+      uses: ./.github/workflows/utils/setup_self_hosted_macos_env_linux_arm64
+      with:
+        python_version: ${{ matrix.python_version.official }}
+
+    - name: Get Cached LLVM Source
+      id: cache-llvm-source
+      uses: actions/cache@v4
+      with:
+        path: mlir/llvm-project
+        key: ${{ runner.os }}-${{ matrix.container_name }}-llvm-${{ needs.constants.outputs.llvm_version }}-generic-source
+        enableCrossOsArchive: True
+        fail-on-cache-miss: True
+
+    - name: Get Cached LLVM Build
+      id: cache-llvm-build
+      uses: actions/cache@v4
+      with:
+        path:  llvm-build
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.container_name }}-llvm-${{ needs.constants.outputs.llvm_version }}-generic-build
+        fail-on-cache-miss: True
+
+    - name: Get Cached MHLO Source
+      id: cache-mhlo-source
+      uses: actions/cache@v4
+      with:
+        path: mlir/mlir-hlo
+        key: ${{ runner.os }}-${{ matrix.container_name }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-source
+        enableCrossOsArchive: True
+        fail-on-cache-miss: True
+
+    - name: Get Cached MHLO Build
+      id: cache-mhlo-build
+      uses: actions/cache@v4
+      with:
+        path: mhlo-build
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.container_name }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-build
+        fail-on-cache-miss: True
+
+    - name: Get Cached Enzyme Source
+      id: cache-enzyme-source
+      uses: actions/cache@v4
+      with:
+        path: mlir/Enzyme
+        key: ${{ runner.os }}-${{ matrix.container_name }}-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source
+        fail-on-cache-miss: True
+
+    - name: Get Cached Enzyme Build
+      id: cache-enzyme-build
+      uses: actions/cache@v4
+      with:
+        path: enzyme-build
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.container_name }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-generic-build
+        fail-on-cache-miss: True
+
+    # Build Catalyst Wheel
+    - name: Build Catalyst Wheel
+      run: |
+        set -x
+        export GCC_VERSION=13
+        docker run --rm --platform linux/aarch64 \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            -v `pwd`:/catalyst \
+            -i ${{ matrix.container_img }} \
+            bash /catalyst/.github/workflows/scripts/linux_arm64/rh8/build_catalyst.sh $GCC_VERSION ${{ matrix.python_version.official }} ${{ matrix.python_version.subversion }} ${{ matrix.python_version.package }} ${{ matrix.python_version.alternative }}
+
+    - name: Upload Wheel Artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: catalyst-linux_arm64-wheel-py-${{ matrix.python_version.official}}.zip
+        path: wheel/
+        retention-days: 14
+
+  test-wheels:
+    needs: [constants, catalyst-linux-wheels-arm64]
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: [{official: "3.10", subversion: "14", package: "python3.10"}]
+        container_img: ["quay.io/pypa/manylinux_2_28_aarch64"]
+        container_name: ["manylinux_2_28_aarch64"]
+
+    # To check all wheels for supported python3 versions
+    name: Test Wheels (Python ${{ matrix.python_version.official }})
+    runs-on:
+      group: 'Office 24th floor M2 Mac'
+
+    steps:
+    - name: Checkout Catalyst repo
+      uses: actions/checkout@v4
+
+    - name: Setup Runner Environment
+      id: setup_env
+      uses: ./.github/workflows/utils/setup_self_hosted_macos_env_linux_arm64
+      with:
+        python_version: ${{ matrix.python_version.official }}
+
+    - name: Download Wheel Artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: catalyst-linux_arm64-wheel-py-${{ matrix.python_version.official }}.zip
+        path: dist
+
+    # Needed for accessing llvm-symbolizer
+    - name: Get Cached LLVM Build
+      id: cache-llvm-build
+      uses: actions/cache@v4
+      with:
+        path:  llvm-build
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.container_name }}-llvm-${{ needs.constants.outputs.llvm_version }}-generic-build
+        fail-on-cache-miss: True
+
+    - name: Run Python Pytest Tests
+      run: |
+        set -x
+        export GCC_VERSION=13
+        docker run --rm --platform linux/aarch64 \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            -v `pwd`:/catalyst \
+            -i ${{ matrix.container_img }} \
+            bash /catalyst/.github/workflows/scripts/linux_arm64/rh8/test_wheels.sh $GCC_VERSION ${{ matrix.python_version.official }} ${{ matrix.python_version.subversion }} ${{ matrix.python_version.package }}