Merge branch 'main' into justinchu/dft

.azure-pipelines/Linux-CI.yml

@@ -66,7 +66,7 @@ jobs:
         export ONNX_ML=1
       fi
       export CMAKE_ARGS="-DONNX_WERROR=ON -DONNX_USE_PROTOBUF_SHARED_LIBS=ON"
-      # enable more sanitizer
+      # Enable more sanitizers
       export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CXX_FLAGS='-fsanitize=undefined -fno-sanitize-recover=all '"
       pip install -e ".[reference]" -v
     displayName: 'Install ONNX and dependencies'
@@ -107,6 +107,16 @@ jobs:
         exit 1
       fi
 
+      pip uninstall -y pillow
+      python onnx/backend/test/cmd_tools.py generate-data --clean
+      git status
+      # Verify test generation without pillow for ImageDecoder, it should directly use frozen data
+      git diff --exit-code -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' ':!*output_*.pb' ':!*input_*.pb'
+      if [ $? -ne 0 ]; then
+        echo "git diff for test generation without pillow returned failures. Please check updated node test files"
+        exit 1
+      fi
+
     displayName: Test backend test data
 
   - script: |

.azure-pipelines/MacOS-CI.yml

@@ -93,4 +93,13 @@ jobs:
         exit 1
       fi
 
+      pip uninstall -y pillow
+      python onnx/backend/test/cmd_tools.py generate-data --clean
+      git status
+      git diff --exit-code -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' ':!*output_*.pb' ':!*input_*.pb'
+      if [ $? -ne 0 ]; then
+        echo "git diff for test generation without pillow returned failures. Please check updated node test files"
+        exit 1
+      fi
+
     displayName: 'Run ONNX Tests'

.azure-pipelines/Windows-CI.yml

@@ -94,6 +94,15 @@ jobs:
         EXIT 1
       )
 
+      pip uninstall -y pillow
+      python onnx/backend/test/cmd_tools.py generate-data --clean
+      git status
+      git diff --exit-code -- . :!onnx/onnx-data.proto :!onnx/onnx-data.proto3 :!*output_*.pb :!*input_*.pb
+      IF NOT %ERRORLEVEL% EQU 0 (
+        @echo "git diff for test generation without pillow returned failures. Please check updated node test files"
+        EXIT 1
+      )
+
       rm -rf .setuptools-cmake-build
       if '$(protobuf_type)' == 'External' (
         conda install -y -c conda-forge libprotobuf=3.20
@@ -108,8 +117,9 @@ jobs:
         set ONNX_BUILD_TESTS=1
         set ONNX_ML=$(onnx_ml)
         set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=ON
+        python -m pip install -r requirements-release.txt
         pip uninstall -y onnx
-        python setup.py install
+        pip install .
 
         pytest
         IF NOT %ERRORLEVEL% EQU 0 (

.github/workflows/auto_update_doc.yml

@@ -15,7 +15,7 @@ jobs:
         contents: write
 
     steps:
-    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       with:
         repository: ${{ github.event.pull_request.head.repo.full_name }}
         # Checkout the branch made in the fork. Will automatically push changes

.github/workflows/check_urls.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
     - name: urls-checker-code
       uses: urlstechie/urlchecker-action@b643b43e2ac605e1475331c7b67247d242b7dce4 # v0.0.34

.github/workflows/clang_tidy_review.yml

@@ -19,7 +19,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       # Pleaes note that it won't cause any failure here; it will only post comments in PRs
       - name: clang-tidy review
         uses: ZedThree/clang-tidy-review@f41163387a7680d93a25c1a9d24d71437c20e16f # v0.13.3

.github/workflows/codeql.yml

@@ -45,7 +45,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
     - name: Set up Python
       uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
       with:
@@ -74,8 +74,7 @@ jobs:
         python -m pip install --quiet -r requirements-release.txt
         git submodule update --init --recursive
         export ONNX_ML=1
-        sudo python setup.py install
-
+        pip install .
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

.github/workflows/lint.yml

@@ -23,7 +23,7 @@ jobs:
     name: Optional Lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: misspell # Check spellings as well
         uses: reviewdog/action-misspell@9257f108197b44e37995c98bea6ee4a5b9ffc3b0 # v1.13.1
         with:
@@ -56,7 +56,7 @@ jobs:
     permissions:
       security-events: write
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Setup Python
         uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:

.github/workflows/manylinux/entrypoint.sh

@@ -42,9 +42,10 @@ $PIP_INSTALL_COMMAND -r requirements-release.txt || { echo "Installing Python re
 
 # Build wheels
 if [ "$GITHUB_EVENT_NAME" == "schedule" ]; then
-    $PYTHON_COMMAND setup.py bdist_wheel --weekly_build || { echo "Building wheels failed."; exit 1; }
+    sed -i 's/name = "onnx"/name = "onnx-weekly"/' 'pyproject.toml'
+    ONNX_PREVIEW_BUILD=1 $PYTHON_COMMAND -m build --wheel || { echo "Building wheels failed."; exit 1; }
 else
-    $PYTHON_COMMAND setup.py bdist_wheel || { echo "Building wheels failed."; exit 1; }
+    $PYTHON_COMMAND -m build --wheel || { echo "Building wheels failed."; exit 1; }
 fi
 
 # Bundle external shared libraries into the wheels

.github/workflows/pages.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - name: Setup Python
         uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
@@ -42,7 +42,7 @@ jobs:
           git submodule update --init --recursive
           export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON"
           export ONNX_ML=1
-          python setup.py install
+          pip install .
       - name: Build Docs
         run: |
           cd docs/docsgen

.github/workflows/release_linux_aarch64.yml

@@ -26,7 +26,7 @@ jobs:
       img: quay.io/pypa/manylinux2014_aarch64
 
     steps:
-    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
     - name: Checkout submodules
       shell: bash
       run: |

.github/workflows/release_linux_x86_64.yml

@@ -23,7 +23,7 @@ jobs:
         architecture: ['x64']
 
     steps:
-    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
     - name: Checkout submodules
       shell: bash
       run: |

.github/workflows/release_mac.yml

@@ -13,7 +13,7 @@ on:
 # Use MACOSX_DEPLOYMENT_TARGET=10.12 to produce compatible wheel
 env:
   MACOSX_DEPLOYMENT_TARGET: 10.12
-  
+
 permissions:
   contents: read
 
@@ -28,7 +28,7 @@ jobs:
         target-architecture: ['x86_64', 'universal2']
 
     steps:
-    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
     - name: Checkout submodules
       shell: bash
       run: |
@@ -51,25 +51,27 @@ jobs:
         CC: "clang"
         CXX: "clang++"
         ONNX_ML: 1
+        CMAKE_OSX_ARCHITECTURES: ${{ matrix.target-architecture == 'x86_64' && 'x86_64' || 'arm64;x86_64' }}
+        # Currently GitHub Action agent is using macos-11, we rename the wheels
+        # to use the MACOSX_DEPLOYMENT_TARGET
+        # Rename e.g. onnx-1.15.0-cp38-cp38-macosx_11_0_x86_64.whl
+        # to onnx-1.15.0-cp38-cp38-macosx_10_12_universal2.whl
+        ONNX_WHEEL_PLATFORM_NAME: macosx_10_12_${{ matrix.target-architecture }}
+        CMAKE_ARGS: "-DONNX_USE_LITE_PROTO=ON"
       run: |
         # Install Protobuf from source
         export NUM_CORES=`sysctl -n hw.logicalcpu`
-        if [ '${{ matrix.target-architecture }}' == 'x86_64' ]; then
-          export CMAKE_OSX_ARCHITECTURES=x86_64
-        elif [ '${{ matrix.target-architecture }}' == 'universal2' ]; then
-          export CMAKE_OSX_ARCHITECTURES='arm64;x86_64'
-        fi
         source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install
-        export CMAKE_ARGS="-DONNX_USE_LITE_PROTO=ON"
-        # Currently GitHub Action agent is using macos-11, use -p to force change the final MACOSX_DEPLOYMENT_TARGET
-        # Change -p if MACOSX_DEPLOYMENT_TARGET is different
         if [ '${{ github.event_name }}' == 'schedule' ]; then
-          python setup.py bdist_wheel -p macosx_10_12_${{ matrix.target-architecture }} --weekly_build
+          sed -i 's/name = "onnx"/name = "onnx-weekly"/' 'pyproject.toml'
+          ONNX_PREVIEW_BUILD=1 python -m build --wheel
         else
-          python setup.py bdist_wheel -p macosx_10_12_${{ matrix.target-architecture }}
+          python -m build --wheel
         fi
-        for file in dist/*.whl; do python -m pip install --upgrade $file; done
 
+        for file in dist/*.whl; do
+          python -m pip install --upgrade $file;
+        done
     - name: Test the installed wheel
       run: |
         pytest
@@ -123,13 +125,14 @@ jobs:
       run: |
         # Build and upload source distribution to PyPI
         git clean -xdf
-        python setup.py sdist --weekly_build
+        sed -i 's/name = "onnx"/name = "onnx-weekly"/' 'pyproject.toml'
+        ONNX_PREVIEW_BUILD=1 python -m build --sdist
         twine upload dist/* --repository-url https://upload.pypi.org/legacy/ -u ${{ secrets.ONNXWEEKLY_USERNAME }} -p ${{ secrets.ONNXWEEKLY_TOKEN }}
 
         # Test weekly source distribution from PyPI
         python -m pip uninstall -y onnx-weekly
         python -m pip install setuptools
-        python -m pip install --use-deprecated=legacy-resolver --no-use-pep517 --no-binary onnx-weekly onnx-weekly
+        python -m pip install --use-deprecated=legacy-resolver --no-binary onnx-weekly onnx-weekly
         pytest
 
     - name: Verify ONNX with ONNX Runtime PyPI package

.github/workflows/release_win.yml

@@ -23,7 +23,7 @@ jobs:
         architecture: ['x64', 'x86']
     steps:
     - name: Checkout ONNX
-      uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       with:
          path: ./onnx
     - name: Checkout ONNX submodules
@@ -50,7 +50,8 @@ jobs:
         python -m pip install -q --upgrade pip
         cd onnx
         if ('${{ matrix.architecture }}' -eq 'x86') {
-          sed -i '' '/google-re2/d' requirements-release.txt
+          echo "Skip installing dependencies for reference, because they don't have prebuilt wheel on x86"
+          sed -i '' '/-r requirements-reference.txt/d' requirements-release.txt
         }
         python -m pip install -q -r requirements-release.txt
 
@@ -70,9 +71,10 @@ jobs:
 
         if ('${{ github.event_name }}' -eq 'schedule') {
           echo "Build weekly PyPI package"
-          python setup.py bdist_wheel --weekly_build
+          (Get-Content -Path 'pyproject.toml') | ForEach-Object { $_ -replace 'name = "onnx"', 'name = "onnx-weekly"' } | Set-Content -Path 'pyproject.toml'
+          ONNX_PREVIEW_BUILD=1 python -m build --wheel
         } else {
-          python setup.py bdist_wheel
+          python -m build --wheel
         }
         Get-ChildItem -Path dist/*.whl | foreach {python -m pip install --upgrade $_.fullname}
 

.github/workflows/reuse.yml

@@ -13,6 +13,6 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
     - name: REUSE Compliance Check
       uses: fsfe/reuse-action@4f2804894b54004c8ed4b8a62b7c649e54a3aa4b # v2.0.0

.github/workflows/scorecard.yml

@@ -32,7 +32,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.1.0
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.1.0
         with:
           persist-credentials: false
 

.github/workflows/stale.yml

@@ -24,7 +24,6 @@ jobs:
     - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-label: 'no-issue-activity'
-        stale-pr-label: 'no-pr-activity'
         days-before-stale: 365
         days-before-close: 21
+        ascending: true

.github/workflows/weekly_mac_ci.yml

@@ -22,7 +22,7 @@ jobs:
     if: github.event_name != 'pull_request' || contains( github.event.pull_request.labels.*.name, 'test ONNX Model Zoo')
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       name: Checkout repo
     - name: Checkout submodules
       shell: bash

.github/workflows/win_no_exception_ci.yml

@@ -22,7 +22,7 @@ jobs:
         architecture: ['x64']
     steps:
     - name: Checkout ONNX
-      uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       with:
          path: ./onnx