Feature/misc fixes

.github/actions/setup-build/action.yml

@@ -9,6 +9,12 @@ inputs:
       but the content is irrelevant.
     required: false
     default: ''
+  torch-version:
+    description: |
+      Additional string to determine wether to test against a stable
+      torch release or against the nightly build
+    required: true
+    default: 'nightly'
 
 runs:
   using: "composite"
@@ -26,13 +32,15 @@ runs:
 
   - name: Install PyTorch nightly depends
     run: |
-      python -m pip install -r pytorch-requirements.txt
+      python -m pip install -r pytorch-${{ inputs.torch-version }}-requirements.txt
       python -m pip install -r build-requirements.txt
     shell: bash
 
   - name: Install prerequisites (Linux)
     if: ${{ runner.os == 'Linux' }}
-    run: sudo apt-get install --yes ccache ninja-build
+    run: |
+      sudo apt-get update
+      sudo apt-get install --yes ccache ninja-build
     shell: bash
 
   - name: Install prerequisites (macOS)

.github/workflows/RollPyTorch.yml

@@ -52,8 +52,8 @@ jobs:
         # Read the version from the downloaded whl file without extracting it
         PT_RELEASE=$(unzip -p torch-*.whl 'torch-*/METADATA' | grep "^Version:" | awk '{ print $2 }' | sed 's/\([^+]*\).*/\1/')
         echo "Found torch release ${PT_RELEASE}"
-        printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt
-        printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt
+        printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-nightly-requirements.txt
+        printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-nightly-requirements.txt
 
         # Read the commit hash from the downloaded whl file without extracting it
         PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | awk '{ print $3 }' | tr -d "'")

.github/workflows/buildAndTest.yml

@@ -2,9 +2,9 @@ name: Build and Test
 
 on:
   pull_request:
-    branches: [ main ]
+    branches: [ feature/misc_fixes ]
   push:
-    branches: [ main ]
+    branches: [ feature/misc_fixes ]
   workflow_dispatch:
 
 # Ensure that only a single job or workflow using the same
@@ -25,9 +25,10 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os-arch: [ubuntu-x86_64, macos-arm64, windows-x86_64]
-        llvm-build: [in-tree, out-of-tree]
-        torch-binary: [ON, OFF]
+        os-arch: [ubuntu-x86_64]
+        llvm-build: [in-tree]
+        torch-binary: [ON]
+        torch-version: [nightly, stable]
         exclude:
           # Exclude llvm in-tree and pytorch source
           - llvm-build: in-tree
@@ -38,17 +39,11 @@ jobs:
           # Exclude macos-arm64 and llvm out-of-tree altogether
           - os-arch: macos-arm64
             llvm-build: out-of-tree
-          - os-arch: windows-x86_64
-            llvm-build: out-of-tree
-        include:
-          # Specify OS versions
-          - os-arch: ubuntu-x86_64
-            os: a100
           - os-arch: macos-arm64
-            os: macos-latest
+            torch-version: stable
           - os-arch: windows-x86_64
-            os: windows-latest
-    runs-on: ${{ matrix.os }}
+            llvm-build: out-of-tree
+    runs-on: ubuntu-latest
 
     steps:
 
@@ -75,6 +70,7 @@ jobs:
       uses: ./.github/actions/setup-build
       with:
         cache-suffix: 'build-${{ matrix.llvm-build }}'
+        torch-version: ${{ matrix.torch-version }}
 
     - name: Set up Visual Studio shell
       if: ${{ matrix.os-arch == 'windows-x86_64' }}
@@ -98,6 +94,7 @@ jobs:
         TM_PACKAGES="${{ matrix.llvm-build }}" \
         TM_USE_PYTORCH_BINARY="${{ matrix.torch-binary }}" \
         TM_PYTORCH_INSTALL_WITHOUT_REBUILD="${{ steps.cache-pytorch.outputs.cache-hit }}" \
+        TORCH_VERSION="${{ matrix.torch-version }}" \
         ./build_tools/python_deploy/build_linux_packages.sh
 
     - name: Configure os-arch='macos-arm64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}'

.github/workflows/buildRelease.yml

@@ -13,11 +13,16 @@ on:
 jobs:
   build_linux:
     name: Manylinux Build
-    runs-on: a100
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: write
+      packages: write
     strategy:
       matrix:
-        package: [ torch-mlir, torch-mlir-core ]
-        py_version: [ cp38-cp38, cp310-cp310, cp311-cp311 ]
+        package: [ torch-mlir ]
+        py_version: [ cp38-cp38 ]
+        torch-version: [stable] # nightly
         exclude:
           - package: torch-mlir-core
             py_version: cp38-cp38
@@ -47,7 +52,11 @@ jobs:
         python -m pip install wheel
         TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }}
         printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version
-        TM_PYTHON_VERSIONS=${{ matrix.py_version }} TM_PACKAGES=${{ matrix.package }} ./build_tools/python_deploy/build_linux_packages.sh
+        TM_SKIP_TESTS=ON \
+        TM_PYTHON_VERSIONS=${{ matrix.py_version }} \
+        TM_PACKAGES=${{ matrix.package }} \
+        TORCH_VERSION="${{ matrix.torch-version }}" \
+        ./build_tools/python_deploy/build_linux_packages.sh
 
     # If we were given a release_id, then upload the package we just built
     # to the github releases page.
@@ -56,67 +65,7 @@ jobs:
       id: upload-release-assets
       uses: dwenegar/upload-release-assets@v1
       env:
-        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
-      with:
-        release_id: ${{ github.event.inputs.release_id }}
-        assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl
-    # Publishing is necessary to make the release visible to `pip`
-    # on the github releases page.
-    - name: Publish Release (if requested)
-      if: github.event.inputs.release_id != ''
-      id: publish_release
-      uses: eregon/publish-release@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
-      with:
-        release_id: ${{ github.event.inputs.release_id }}
-    - name: Create dist directory
-      if: github.event.inputs.release_id != ''
-      run: mkdir dist
-    - name: Copy releases to publish to dist directory
-      if: github.event.inputs.release_id != ''
-      run: cp build_tools/python_deploy/wheelhouse/torch_mlir*.whl dist/
-
-    # Wheels must be published from a linux environment.
-    #
-    # See https://github.com/pypa/gh-action-pypi-publish/discussions/15
-    - name: Store the binary wheel
-      uses: actions/upload-artifact@v2
-      with:
-        name: wheels
-        path: dist
-
-  build_macos:
-    name: MacOS Build
-    runs-on: macos-latest
-    strategy:
-      matrix:
-        package: [ torch-mlir, torch-mlir-core ]
-    steps:
-    - name: Get torch-mlir
-      uses: actions/checkout@v3
-      with:
-        submodules: 'true'
-    - uses: ./.github/actions/setup-build
-      with:
-        cache-suffix: 'release'
-    - name: Build Python wheels and smoke test.
-      run: |
-        cd $GITHUB_WORKSPACE
-        python -m pip install wheel
-        TM_PACKAGE_VERSION=${{ github.event.inputs.python_package_version }}
-        printf "TORCH_MLIR_PYTHON_PACKAGE_VERSION=%s\n" $TM_PACKAGE_VERSION > ./torch_mlir_package_version
-        sudo ./build_tools/python_deploy/install_macos_deps.sh
-        packages=${{ matrix.package }} TORCH_MLIR_PYTHON_VERSIONS="3.11" ./build_tools/python_deploy/build_macos_packages.sh
-
-    # If we were given a release_id, then upload the package we just built
-    # to the github releases page.
-    - name: Upload Release Assets (if requested)
-      if: github.event.inputs.release_id != ''
-      id: upload-release-assets
-      uses: dwenegar/upload-release-assets@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
       with:
         release_id: ${{ github.event.inputs.release_id }}
         assets_path: ./build_tools/python_deploy/wheelhouse/torch*.whl
@@ -127,7 +76,7 @@ jobs:
       id: publish_release
       uses: eregon/publish-release@v1
       env:
-        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
       with:
         release_id: ${{ github.event.inputs.release_id }}
     - name: Create dist directory
@@ -146,82 +95,14 @@ jobs:
         name: wheels
         path: dist
 
-  build_windows:
-    name: Windows Build
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        package: [ torch-mlir, torch-mlir-core ]
-    steps:
-    - name: Get torch-mlir
-      uses: actions/checkout@v3
-      with:
-        submodules: 'true'
-    - uses: ./.github/actions/setup-build
-      with:
-        cache-suffix: 'release'
-    - name: Set up Visual Studio shell
-      uses: egor-tensin/vs-shell@v2
-      with:
-        arch: x64
-    - name: Build Python wheels and smoke test.
-      shell: pwsh
-      run: |
-        if ( "${{ matrix.package }}" -eq "torch-mlir-core" )
-        {
-          $env:TORCH_MLIR_ENABLE_JIT_IR_IMPORTER='0'
-          $env:TORCH_MLIR_ENABLE_ONLY_MLIR_PYTHON_BINDINGS='1'
-        } else {
-          $env:TORCH_MLIR_ENABLE_JIT_IR_IMPORTER='1'
-          $env:TORCH_MLIR_ENABLE_ONLY_MLIR_PYTHON_BINDINGS='0'
-        }
-        $env:TORCH_MLIR_PYTHON_PACKAGE_VERSION = '${{ github.event.inputs.python_package_version }}'
-        ./build_tools/python_deploy/build_windows.ps1
-
-    # If we were given a release_id, then upload the package we just built
-    # to the github releases page.
-    - name: Upload Release Assets (if requested)
-      if: github.event.inputs.release_id != ''
-      id: upload-release-assets
-      uses: dwenegar/upload-release-assets@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
-      with:
-        release_id: ${{ github.event.inputs.release_id }}
-        assets_path: ./wheelhouse/torch*.whl
-    # Publishing is necessary to make the release visible to `pip`
-    # on the github releases page.
-    - name: Publish Release (if requested)
-      if: github.event.inputs.release_id != ''
-      id: publish_release
-      uses: eregon/publish-release@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
-      with:
-        release_id: ${{ github.event.inputs.release_id }}
-    - name: Create dist directory
-      if: github.event.inputs.release_id != ''
-      run: mkdir dist
-      continue-on-error: true
-    - name: Copy releases to publish to dist directory
-      if: github.event.inputs.release_id != ''
-      run: cp ./wheelhouse/torch_mlir*.whl dist/
-
-    # Wheels must be published from a linux environment.
-    #
-    # See https://github.com/pypa/gh-action-pypi-publish/discussions/15
-    - name: Store the binary wheel
-      uses: actions/upload-artifact@v2
-      with:
-        name: wheels
-        path: dist
-
   publish_releases:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      actions: write
+      packages: write
     needs:
     - build_linux
-    - build_macos
-    - build_windows
 
     # Publish even if one of the builds failed
     if: ${{ always() }}
@@ -231,7 +112,7 @@ jobs:
       uses: benc-uk/workflow-dispatch@v1
       with:
         workflow: Publish releases page
-        token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+        token: ${{ secrets.GITHUB_TOKEN  }}
 
     # Wheels must be published from a linux environment.
     #

.github/workflows/gh-pages-releases.yml

@@ -8,9 +8,11 @@ jobs:
   scrape_and_publish_releases:
     name: "Scrape and publish releases"
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
 
     # Don't run this in everyone's forks.
-    if: github.repository == 'llvm/torch-mlir'
+    if: github.repository == 'xilinx/torch-mlir'
 
     steps:
       - name: Prepare workspace
@@ -20,10 +22,8 @@ jobs:
           sudo rm -rf $GITHUB_WORKSPACE/*
       - name: Checking out repository
         uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
       - name: Run scrape releases script
-        run: python ./build_tools/scrape_releases.py llvm torch-mlir > /tmp/index.html
+        run: python ./build_tools/scrape_releases.py xilinx torch-mlir > /tmp/index.html
         shell: bash
       - run: git fetch --all
       - run: git switch github-pages

.github/workflows/oneshotSnapshotPackage.yml

@@ -8,18 +8,19 @@ jobs:
     name: "Tag snapshot release"
     runs-on: ubuntu-latest
     # Don't run this in everyone's forks.
-    if: github.repository == 'llvm/torch-mlir'
+    #if: github.repository == 'llvm/torch-mlir'
     steps:
       - name: Prepare workspace
         run: |
           # Clear the workspace directory so that we don't run into errors about
           # existing lock files.
           sudo rm -rf $GITHUB_WORKSPACE/*
 
-      - name: Checking out repository
+      - name: Checkout torch-mlir
         uses: actions/checkout@v3
         with:
-          token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}
+            submodules: 'true'
+            fetch-depth: 0
 
       - name: Compute version
         run: |