Merge remote-tracking branch 'origin/main' into onnxopt

.azure-pipelines/Linux-CI.yml

@@ -4,34 +4,39 @@ trigger:
 jobs:
 - job: 'Test'
   pool:
-    vmImage: 'Ubuntu-18.04'
+    vmImage: 'Ubuntu-20.04'
   strategy:
     matrix:
-      py39-ml-debug:
-        python.version: '3.9'
+      py311-ml-debug:
+        python.version: '3.11'
         onnx_ml: 1
-        onnx_debug: 1
+        build_type: 'Debug'
         documentation: 0
-      py38:
-        python.version: '3.8'
+        protobuf_type: 'External'
+      py310-InternalProtobuf:
+        python.version: '3.10'
         onnx_ml: 0
-        onnx_debug: 0
+        build_type: 'Release'
         documentation: 0
-      py38-ml:
-        python.version: '3.8'
+        protobuf_type: 'Internal'
+      py39-ml:
+        python.version: '3.9'
         onnx_ml: 1
-        onnx_debug: 0
+        build_type: 'Release'
         documentation: 1
-      py37:
-        python.version: '3.7'
+        protobuf_type: 'External'
+      py38-InternalProtobuf:
+        python.version: '3.8'
         onnx_ml: 0
-        onnx_debug: 0
+        build_type: 'Release'
         documentation: 0
-      py37-ml:
-        python.version: '3.7'
+        protobuf_type: 'Internal'
+      py38-ml:
+        python.version: '3.8'
         onnx_ml: 1
-        onnx_debug: 0
+        build_type: 'Release'
         documentation: 0
+        protobuf_type: 'External'
     maxParallel: 6
 
   steps:
@@ -45,32 +50,43 @@ jobs:
       python -m virtualenv venv
       source venv/bin/activate
 
-      sudo apt-get install libprotobuf-dev protobuf-compiler
-      python -m pip install -q --upgrade pip
-      python -m pip install -q -r requirements-release.txt
+      if [ '$(protobuf_type)' == 'External' ]; then
+        sudo apt-get install libprotobuf-dev protobuf-compiler
+      elif [ '$(protobuf_type)' == 'Internal' ]; then
+        echo "Use the internal protobuf build"
+      fi
+
+      python -m pip install --upgrade pip
+      python -m pip install -r requirements-release.txt
+      # still test protobuf==3.20.2 at least in a CI
+      python -m pip install protobuf==3.20.2
 
       sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y --no-install-recommends dos2unix
 
       git submodule update --init --recursive
       export ONNX_BUILD_TESTS=1
-      if [ '$(onnx_debug)' == '1' ]; then
+      if [ '$(build_type)' == 'Debug' ]; then
         export DEBUG=1
       fi
       if [ '$(onnx_ml)' == '1' ]; then
         export ONNX_ML=1
       fi
       export CMAKE_ARGS="-DONNX_WERROR=ON -DONNX_USE_PROTOBUF_SHARED_LIBS=ON"
-      export ONNX_NAMESPACE=ONNX_NAMESPACE_FOO_BAR_FOR_CI
-      python setup.py --quiet install
+      # enable more sanitizer
+      export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CXX_FLAGS='-fsanitize=undefined -fno-sanitize-recover=all '"
+      pip install -e . -v
     displayName: 'Install ONNX and dependencies'
 
   - script: |
       source venv/bin/activate
 
-      # check line endings to be UNIX
-      find . -type f -regextype posix-extended -regex '.*\.(py|cpp|md|h|cc|proto|proto3|in)' | xargs dos2unix --quiet
-      git status
-      git diff --exit-code
+      pytest -sv
+      if [ $? -ne 0 ]; then
+        echo "pytest failed"
+        exit 1
+      fi
+
+      python -m pip install -r requirements-dev.txt
 
       # check auto-gen files up-to-date
       python onnx/defs/gen_doc.py
@@ -91,12 +107,6 @@ jobs:
       ! grep -R --include='*.cc' --include='*.h' 'namespace onnx' .
       ! grep -R --include='*.cc' --include='*.h' 'onnx::' .
 
-      pytest
-      if [ $? -ne 0 ]; then
-        echo "pytest failed"
-        exit 1
-      fi
-
       # onnx c++ API tests
       export LD_LIBRARY_PATH="./.setuptools-cmake-build/:$LD_LIBRARY_PATH"
       ./.setuptools-cmake-build/onnx_gtests
@@ -130,7 +140,7 @@ jobs:
       if [ '$(documentation)' == '1' ]; then
         source venv/bin/activate
         pip install -r docs/docsgen/source/requirements.txt
-        cd docs/docsgen && make text -j auto
+        cd docs/docsgen && make text
       fi
     displayName: Test documentation
     continueOnError: true  # the documentation generates errors due to operators documentation

.azure-pipelines/MacOS-CI.yml

@@ -7,30 +7,35 @@ jobs:
     vmImage: 'macOS-11'
   strategy:
     matrix:
-      py39:
-        python.version: '3.9'
+      py311:
+        python.version: '3.11'
         onnx_ml: 0
-        onnx_debug: 0
+        build_type: 'Release'
+        protobuf_type: 'External'
         onnx_lite: 0
-      py38:
-        python.version: '3.8'
+      py310-InternalProtobuf:
+        python.version: '3.10'
         onnx_ml: 0
-        onnx_debug: 0
+        build_type: 'Release'
+        protobuf_type: 'Internal'
         onnx_lite: 1
-      py38-ml:
-        python.version: '3.8'
+      py39-ml:
+        python.version: '3.9'
         onnx_ml: 1
-        onnx_debug: 0
+        build_type: 'Release'
+        protobuf_type: 'External'
         onnx_lite: 1
-      py38-ml-debug:
-        python.version: '3.8'
+      py39-ml-debug-InternalProtobuf:
+        python.version: '3.9'
         onnx_ml: 1
-        onnx_debug: 1
+        build_type: 'Debug'
+        protobuf_type: 'Internal'
         onnx_lite: 0
-      py37-ml-debug:
-        python.version: '3.7'
+      py38-ml-debug:
+        python.version: '3.8'
         onnx_ml: 1
-        onnx_debug: 1
+        build_type: 'Debug'
+        protobuf_type: 'External'
         onnx_lite: 0
     maxParallel: 6
 
@@ -42,16 +47,18 @@ jobs:
   - script: |
       # Install Protobuf from source
       export NUM_CORES=`sysctl -n hw.logicalcpu`
-      if [ '$(onnx_debug)' == '1' ]; then
+      if [ '$(build_type)' == 'Debug' ]; then
         export DEBUG=1
-        source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install Debug
-      else
-        source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install
+      fi
+      if [ '$(protobuf_type)' == 'External' ]; then
+        source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install $(build_type)
+      elif [ '$(protobuf_type)' == 'Internal' ]; then
+        echo "Use the internal protobuf build"
       fi
 
       git submodule update --init --recursive
-      python -m pip install -q --upgrade pip
-      python -m pip install -q -r requirements-release.txt
+      python -m pip install --upgrade pip
+      python -m pip install -r requirements-release.txt
 
       if [ '$(onnx_ml)' == '1' ]; then
         export ONNX_ML=1
@@ -61,11 +68,16 @@ jobs:
       if [ '$(onnx_lite)' == '1' ]; then
         export CMAKE_ARGS="${CMAKE_ARGS} -DONNX_USE_LITE_PROTO=ON"
       fi
-      export ONNX_NAMESPACE=ONNX_NAMESPACE_FOO_BAR_FOR_CI
-      python setup.py -q install
+      pip install -e . -v
     displayName: 'Install dependencies and ONNX'
 
   - script: |
+      pytest
+      if [ $? -ne 0 ]; then
+        echo "pytest failed"
+        exit 1
+      fi
+
       # check auto-gen files up-to-date
       python onnx/defs/gen_doc.py
       python onnx/gen_proto.py -l
@@ -85,12 +97,6 @@ jobs:
       ! grep -R --include='*.cc' --include='*.h' 'namespace onnx' .
       ! grep -R --include='*.cc' --include='*.h' 'onnx::' .
 
-      pytest
-      if [ $? -ne 0 ]; then
-        echo "pytest failed"
-        exit 1
-      fi
-
       # onnx c++ API tests
       export LD_LIBRARY_PATH="./.setuptools-cmake-build/:$LD_LIBRARY_PATH"
       ./.setuptools-cmake-build/onnx_gtests

.azure-pipelines/Windows-CI.yml

@@ -8,26 +8,31 @@ jobs:
     vmImage: 'windows-2019'
   strategy:
     matrix:
-      py39:
-        python.version: '3.9'
+      py311:
+        python.version: '3.11'
         onnx_ml: 0
         onnx_verify_proto: 0
-      py39_verify_proto:
-        python.version: '3.9'
+        protobuf_type: 'External'
+      py310_verify_proto_internal_protobuf:
+        python.version: '3.10'
         onnx_ml: 0
         onnx_verify_proto: 1
-      py38:
-        python.version: '3.8'
+        protobuf_type: 'Internal'
+      py39:
+        python.version: '3.9'
         onnx_ml: 0
         onnx_verify_proto: 0
-      py37:
-        python.version: '3.7'
+        protobuf_type: 'External'
+      py38_internal_protobuf:
+        python.version: '3.8'
         onnx_ml: 0
         onnx_verify_proto: 0
-      py37_onnx_ml:
-        python.version: '3.7'
+        protobuf_type: 'Internal'
+      py38_onnx_ml:
+        python.version: '3.8'
         onnx_ml: 1
         onnx_verify_proto: 0
+        protobuf_type: 'External'
     maxParallel: 4
 
   steps:
@@ -41,30 +46,49 @@ jobs:
 
   - script: |
       conda create --yes --quiet --name py$(python.version) python=$(python.version)
-      conda install -n py$(python.version) -y -c conda-forge numpy libprotobuf=3.20
+      if '$(protobuf_type)' == 'External' (
+        conda install -n py$(python.version) -y -c conda-forge numpy libprotobuf==3.21.5
+      ) else (
+        conda install -n py$(python.version) -y -c conda-forge numpy
+      )
     displayName: Create Anaconda environment
 
   - script: |
       call activate py$(python.version)
-      python -m pip install -q --upgrade pip
-      python -m pip install -q -r requirements-release.txt
+      python -m pip install --upgrade pip
+      python -m pip install -r requirements-release.txt
 
       git submodule update --init --recursive
       set ONNX_BUILD_TESTS=1
       set ONNX_ML=$(onnx_ml)
-      set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=ON
+      if '$(protobuf_type)' == 'External' (
+        set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=ON
+      ) else (
+        REM Disable ONNX_WERROR since the new protobuf version has build warnings that are treated as errors.
+        set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=OFF
+      )
 
-      python setup.py -q install
+      pip install -e . -v
       pytest
       IF NOT %ERRORLEVEL% EQU 0 (
         @echo "pytest failed"
         EXIT 1
       )
 
+      python -m pip install onnxruntime
+      set ORT_MAX_IR_SUPPORTED_VERSION=8
+      set ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3
+      set ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=18
+      pytest
+      IF NOT %ERRORLEVEL% EQU 0 (
+        @echo "pytest failed when testing onnx with onnxruntime"
+        EXIT 1
+      )
+
       python onnx/defs/gen_doc.py
       python onnx/gen_proto.py -l
       python onnx/gen_proto.py -l --ml
-      
+
       git diff --exit-code -- . :(exclude)onnx/onnx-data.proto :(exclude)onnx/onnx-data.proto3
       IF NOT %ERRORLEVEL% EQU 0 (
         @echo "git diff returned failures"
@@ -86,5 +110,22 @@ jobs:
 
       rm -rf .setuptools-cmake-build
 
-
+      conda install -y -c conda-forge libprotobuf=3.20
     displayName: Install and test ONNX
+
+  - script: |
+      call activate py$(python.version)
+      python -m pip install protobuf==3.20.2
+
+      set ONNX_BUILD_TESTS=1
+      set ONNX_ML=$(onnx_ml)
+      set CMAKE_ARGS=-DONNX_USE_PROTOBUF_SHARED_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DONNX_WERROR=ON
+      pip uninstall -y onnx
+      python setup.py install
+
+      pytest
+      IF NOT %ERRORLEVEL% EQU 0 (
+        @echo "pytest failed when testing onnx with libprotobuf=3.20"
+        EXIT 1
+      )
+    displayName: Test Protobuf 3.20 from conda-forge

.clang-tidy

@@ -16,7 +16,6 @@ Checks: >-
   clang-analyzer-alpha.cplusplus.*,
   clang-analyzer-alpha.security.*,
   cppcoreguidelines-avoid-goto,
-  cppcoreguidelines-init-variables,
   cppcoreguidelines-interfaces-global-init,
   cppcoreguidelines-no-malloc,
   cppcoreguidelines-prefer-member-initializer,

.editorconfig

@@ -0,0 +1,7 @@
+root = true
+
+[*]
+
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space

.github/ISSUE_TEMPLATE/bug.md

@@ -15,9 +15,9 @@ assignees: ''
 <!-- Please describe the bug clearly and concisely -->
 
 ### System information
-<!-- 
-- OS Platform and Distribution (*e.g. Linux Ubuntu 16.04*): 
-- ONNX version (*e.g. 1.7*):  
+<!--
+- OS Platform and Distribution (*e.g. Linux Ubuntu 20.04*):
+- ONNX version (*e.g. 1.13*):
 - Python version:
 - GCC/Compiler version (if compiling from source):
 - CMake version: