Merge tag 'v1.15.0' into HEAD

daquexian · Nov 2, 2023 · 46c3ab3 · 46c3ab3
2 parents 72f2355 + b86cc54
commit 46c3ab3
Show file tree

Hide file tree

Showing 830 changed files with 24,679 additions and 6,329 deletions.
diff --git a/.azure-pipelines/Linux-CI.yml b/.azure-pipelines/Linux-CI.yml
@@ -1,43 +1,44 @@
+schedules:
+- cron: '0 0 * * *'
+  displayName: Nightly Linux CI in main branch
+  branches:
+    include:
+    - main
+
 trigger:
-- main
+- gh-readonly-queue/**
 
 jobs:
 - job: 'Test'
   pool:
     vmImage: 'Ubuntu-20.04'
   strategy:
     matrix:
-      py311-ml-debug:
+      py311-ml-debug-external-protobuf:
         python.version: '3.11'
         onnx_ml: 1
         build_type: 'Debug'
         documentation: 0
         protobuf_type: 'External'
-      py310-InternalProtobuf:
+      py310-internal-protobuf:
         python.version: '3.10'
         onnx_ml: 0
         build_type: 'Release'
         documentation: 0
         protobuf_type: 'Internal'
-      py39-ml:
+      py39-ml-doc-external-protobuf:
         python.version: '3.9'
         onnx_ml: 1
         build_type: 'Release'
         documentation: 1
         protobuf_type: 'External'
-      py38-InternalProtobuf:
+      py38-internal-protobuf:
         python.version: '3.8'
         onnx_ml: 0
         build_type: 'Release'
         documentation: 0
         protobuf_type: 'Internal'
-      py38-ml:
-        python.version: '3.8'
-        onnx_ml: 1
-        build_type: 'Release'
-        documentation: 0
-        protobuf_type: 'External'
-    maxParallel: 6
+    maxParallel: 4
 
   steps:
   - task: UsePythonVersion@0
@@ -72,9 +73,9 @@ jobs:
         export ONNX_ML=1
       fi
       export CMAKE_ARGS="-DONNX_WERROR=ON -DONNX_USE_PROTOBUF_SHARED_LIBS=ON"
-      # enable more sanitizer
+      # Enable more sanitizers
       export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CXX_FLAGS='-fsanitize=undefined -fno-sanitize-recover=all '"
-      pip install -e . -v
+      pip install -e ".[reference]" -v
     displayName: 'Install ONNX and dependencies'
 
   - script: |
@@ -86,27 +87,6 @@ jobs:
         exit 1
       fi
 
-      python -m pip install -r requirements-dev.txt
-
-      # check auto-gen files up-to-date
-      python onnx/defs/gen_doc.py
-      python onnx/gen_proto.py -l
-      python onnx/gen_proto.py -l --ml
-      python onnx/backend/test/stat_coverage.py
-
-      git status
-      git diff --exit-code -- . ':(exclude)onnx/onnx-data.proto' ':(exclude)onnx/onnx-data.proto3'
-      if [ $? -ne 0 ]; then
-        echo "git diff returned failures"
-        exit 1
-      fi
-
-      # Do not hardcode onnx's namespace in the c++ source code, so that
-      # other libraries who statically link with onnx can hide onnx symbols
-      # in a private namespace.
-      ! grep -R --include='*.cc' --include='*.h' 'namespace onnx' .
-      ! grep -R --include='*.cc' --include='*.h' 'onnx::' .
-
       # onnx c++ API tests
       export LD_LIBRARY_PATH="./.setuptools-cmake-build/:$LD_LIBRARY_PATH"
       ./.setuptools-cmake-build/onnx_gtests
@@ -134,6 +114,16 @@ jobs:
         exit 1
       fi
 
+      pip uninstall -y pillow
+      python onnx/backend/test/cmd_tools.py generate-data --clean
+      git status
+      # Verify test generation without pillow for ImageDecoder, it should directly use frozen data
+      git diff --exit-code -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' ':!*output_*.pb' ':!*input_*.pb'
+      if [ $? -ne 0 ]; then
+        echo "git diff for test generation without pillow returned failures. Please check updated node test files"
+        exit 1
+      fi
+
     displayName: Test backend test data
 
   - script: |

diff --git a/.azure-pipelines/MacOS-CI.yml b/.azure-pipelines/MacOS-CI.yml
@@ -1,43 +1,44 @@
+schedules:
+- cron: '0 0 * * *'
+  displayName: Nightly MacOS CI in main branch
+  branches:
+    include:
+    - main
+
 trigger:
-- main
+- gh-readonly-queue/**
 
 jobs:
 - job: 'Test'
   pool:
     vmImage: 'macOS-11'
   strategy:
     matrix:
-      py311:
+      py311-external-protobuf:
         python.version: '3.11'
         onnx_ml: 0
         build_type: 'Release'
         protobuf_type: 'External'
         onnx_lite: 0
-      py310-InternalProtobuf:
+      py310-lite-internal-protobuf:
         python.version: '3.10'
         onnx_ml: 0
         build_type: 'Release'
         protobuf_type: 'Internal'
         onnx_lite: 1
-      py39-ml:
+      py39-ml-lite-external-protobuf:
         python.version: '3.9'
         onnx_ml: 1
         build_type: 'Release'
         protobuf_type: 'External'
         onnx_lite: 1
-      py39-ml-debug-InternalProtobuf:
-        python.version: '3.9'
-        onnx_ml: 1
-        build_type: 'Debug'
-        protobuf_type: 'Internal'
-        onnx_lite: 0
-      py38-ml-debug:
+      py38-ml-debug-external-protobuf:
         python.version: '3.8'
         onnx_ml: 1
         build_type: 'Debug'
         protobuf_type: 'External'
         onnx_lite: 0
-    maxParallel: 6
+    maxParallel: 4
 
   steps:
   - task: UsePythonVersion@0
@@ -68,7 +69,7 @@ jobs:
       if [ '$(onnx_lite)' == '1' ]; then
         export CMAKE_ARGS="${CMAKE_ARGS} -DONNX_USE_LITE_PROTO=ON"
       fi
-      pip install -e . -v
+      pip install -e ".[reference]" -v
     displayName: 'Install dependencies and ONNX'
 
   - script: |
@@ -78,25 +79,16 @@ jobs:
         exit 1
       fi
 
-      # check auto-gen files up-to-date
-      python onnx/defs/gen_doc.py
-      python onnx/gen_proto.py -l
-      python onnx/gen_proto.py -l --ml
-      python onnx/backend/test/stat_coverage.py
-
-      git status
-      git diff --exit-code  -- . ':(exclude)onnx/onnx-data.proto' ':(exclude)onnx/onnx-data.proto3'
+      python -m pip install onnxruntime
+      export ORT_MAX_IR_SUPPORTED_VERSION=8
+      export ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3
+      export ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=19
+      pytest
       if [ $? -ne 0 ]; then
-        echo "git diff returned failures"
+        echo "pytest failed when testing onnx with onnxruntime"
         exit 1
       fi
 
-      # Do not hardcode onnx's namespace in the c++ source code, so that
-      # other libraries who statically link with onnx can hide onnx symbols
-      # in a private namespace.
-      ! grep -R --include='*.cc' --include='*.h' 'namespace onnx' .
-      ! grep -R --include='*.cc' --include='*.h' 'onnx::' .
-
       # onnx c++ API tests
       export LD_LIBRARY_PATH="./.setuptools-cmake-build/:$LD_LIBRARY_PATH"
       ./.setuptools-cmake-build/onnx_gtests
@@ -118,4 +110,19 @@ jobs:
         exit 1
       fi
 
+      pip uninstall -y pillow
+      python onnx/backend/test/cmd_tools.py generate-data --clean
+      git status
+      git diff --exit-code -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' ':!*output_*.pb' ':!*input_*.pb'
+      if [ $? -ne 0 ]; then
+        echo "git diff for test generation without pillow returned failures. Please check updated node test files"
+        exit 1
+      fi
+      # Internal Protobuf won't have other untrack files like protobuf/
+      if [ '$(protobuf_type)' == 'Internal' ]; then
+        if [[ $(git ls-files --others --exclude-standard) ]]; then
+          echo "Some test-generated files not included in the PR. Did you forget to add any test-generated files?"
+          exit 1
+        fi
+      fi
     displayName: 'Run ONNX Tests'
diff --git a/.azure-pipelines/Windows-CI.yml b/.azure-pipelines/Windows-CI.yml
@@ -1,5 +1,12 @@
+schedules:
+- cron: '0 0 * * *'
+  displayName: Nightly Windows CI in main branch
+  branches:
+    include:
+    - main
+
 trigger:
-- main
+- gh-readonly-queue/**
 
 jobs:
 
@@ -8,31 +15,26 @@ jobs:
     vmImage: 'windows-2019'
   strategy:
     matrix:
-      py311:
+      py311-ml-external-protobuf:
         python.version: '3.11'
-        onnx_ml: 0
+        onnx_ml: 1
         onnx_verify_proto: 0
         protobuf_type: 'External'
       py310_verify_proto_internal_protobuf:
         python.version: '3.10'
         onnx_ml: 0
         onnx_verify_proto: 1
         protobuf_type: 'Internal'
-      py39:
+      py39_ml_external_protobuf:
         python.version: '3.9'
-        onnx_ml: 0
+        onnx_ml: 1
         onnx_verify_proto: 0
         protobuf_type: 'External'
-      py38_internal_protobuf:
+      py38_verify_proto_internal_protobuf:
         python.version: '3.8'
         onnx_ml: 0
-        onnx_verify_proto: 0
+        onnx_verify_proto: 1
         protobuf_type: 'Internal'
-      py38_onnx_ml:
-        python.version: '3.8'
-        onnx_ml: 1
-        onnx_verify_proto: 0
-        protobuf_type: 'External'
     maxParallel: 4
 
   steps:
@@ -61,71 +63,67 @@ jobs:
       git submodule update --init --recursive
       set ONNX_BUILD_TESTS=1
       set ONNX_ML=$(onnx_ml)
+      set ONNX_VERIFY_PROTO_3=$(onnx_verify_proto)
       if '$(protobuf_type)' == 'External' (
         set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=ON
       ) else (
         REM Disable ONNX_WERROR since the new protobuf version has build warnings that are treated as errors.
         set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=OFF
       )
 
-      pip install -e . -v
+      pip install -e ".[reference]" -v
       pytest
       IF NOT %ERRORLEVEL% EQU 0 (
         @echo "pytest failed"
         EXIT 1
       )
 
-      python -m pip install onnxruntime
-      set ORT_MAX_IR_SUPPORTED_VERSION=8
-      set ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3
-      set ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=18
-      pytest
+      python onnx/backend/test/cmd_tools.py generate-data --clean
+      git status
+      git diff --exit-code -- . :!onnx/onnx-data.proto :!onnx/onnx-data.proto3 :!*output_*.pb :!*input_*.pb
       IF NOT %ERRORLEVEL% EQU 0 (
-        @echo "pytest failed when testing onnx with onnxruntime"
+        @echo "git diff for test generation returned failures. Please check updated node test files"
         EXIT 1
       )
-
-      python onnx/defs/gen_doc.py
-      python onnx/gen_proto.py -l
-      python onnx/gen_proto.py -l --ml
-
-      git diff --exit-code -- . :(exclude)onnx/onnx-data.proto :(exclude)onnx/onnx-data.proto3
+      git diff --exit-code --diff-filter=ADR -- . :!onnx/onnx-data.proto :!onnx/onnx-data.proto3
       IF NOT %ERRORLEVEL% EQU 0 (
-        @echo "git diff returned failures"
+        @echo "Test generation returned failures. Please check the number of node test files (input_*.pb or output_*.pb)."
         EXIT 1
       )
 
+      pip uninstall -y pillow
       python onnx/backend/test/cmd_tools.py generate-data --clean
       git status
       git diff --exit-code -- . :!onnx/onnx-data.proto :!onnx/onnx-data.proto3 :!*output_*.pb :!*input_*.pb
       IF NOT %ERRORLEVEL% EQU 0 (
-        @echo "git diff for test generation returned failures. Please check updated node test files"
-        EXIT 1
-      )
-      git diff --exit-code --diff-filter=ADR -- . :!onnx/onnx-data.proto :!onnx/onnx-data.proto3
-      IF NOT %ERRORLEVEL% EQU 0 (
-        @echo "Test generation returned failures. Please check the number of node test files (input_*.pb or output_*.pb)."
+        @echo "git diff for test generation without pillow returned failures. Please check updated node test files"
         EXIT 1
       )
 
       rm -rf .setuptools-cmake-build
-
-      conda install -y -c conda-forge libprotobuf=3.20
+      if '$(protobuf_type)' == 'External' (
+        conda install -y -c conda-forge libprotobuf=3.20
+      )
     displayName: Install and test ONNX
 
   - script: |
-      call activate py$(python.version)
-      python -m pip install protobuf==3.20.2
-
-      set ONNX_BUILD_TESTS=1
-      set ONNX_ML=$(onnx_ml)
-      set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=ON
-      pip uninstall -y onnx
-      python setup.py install
+      if '$(protobuf_type)' == 'External' (
+        call activate py$(python.version)
+        python -m pip install protobuf==3.20.2
 
-      pytest
-      IF NOT %ERRORLEVEL% EQU 0 (
-        @echo "pytest failed when testing onnx with libprotobuf=3.20"
-        EXIT 1
+        set ONNX_BUILD_TESTS=1
+        set ONNX_ML=$(onnx_ml)
+        set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=ON
+        python -m pip install -r requirements-release.txt
+        pip uninstall -y onnx
+        pip install .
+
+        pytest
+        IF NOT %ERRORLEVEL% EQU 0 (
+          @echo "pytest failed when testing onnx with libprotobuf=3.20"
+          EXIT 1
+        )
+      ) else (
+        REM Skip the test for externally installing lower version of Protobuf.
       )
     displayName: Test Protobuf 3.20 from conda-forge