Merge branch 'main' into opt-to-upstream

daquexian · Nov 15, 2023 · e1f896d · e1f896d
2 parents 1a19bdb + a5f13c4
commit e1f896d
Show file tree

Hide file tree

Showing 2,206 changed files with 72,253 additions and 14,381 deletions.
diff --git a/.azure-pipelines/Linux-CI.yml b/.azure-pipelines/Linux-CI.yml
@@ -1,38 +1,44 @@
+schedules:
+- cron: '0 0 * * *'
+  displayName: Nightly Linux CI in main branch
+  branches:
+    include:
+    - main
+
 trigger:
-- main
+- gh-readonly-queue/**
 
 jobs:
 - job: 'Test'
   pool:
-    vmImage: 'Ubuntu-18.04'
+    vmImage: 'Ubuntu-20.04'
   strategy:
     matrix:
-      py39-ml-debug:
-        python.version: '3.9'
+      py311-ml-debug-external-protobuf:
+        python.version: '3.11'
         onnx_ml: 1
-        onnx_debug: 1
+        build_type: 'Debug'
         documentation: 0
-      py38:
-        python.version: '3.8'
+        protobuf_type: 'External'
+      py310-internal-protobuf:
+        python.version: '3.10'
         onnx_ml: 0
-        onnx_debug: 0
+        build_type: 'Release'
         documentation: 0
-      py38-ml:
-        python.version: '3.8'
+        protobuf_type: 'Internal'
+      py39-ml-doc-external-protobuf:
+        python.version: '3.9'
         onnx_ml: 1
-        onnx_debug: 0
+        build_type: 'Release'
         documentation: 1
-      py37:
-        python.version: '3.7'
+        protobuf_type: 'External'
+      py38-internal-protobuf:
+        python.version: '3.8'
         onnx_ml: 0
-        onnx_debug: 0
-        documentation: 0
-      py37-ml:
-        python.version: '3.7'
-        onnx_ml: 1
-        onnx_debug: 0
+        build_type: 'Release'
         documentation: 0
-    maxParallel: 6
+        protobuf_type: 'Internal'
+    maxParallel: 4
 
   steps:
   - task: UsePythonVersion@0
@@ -45,53 +51,37 @@ jobs:
       python -m virtualenv venv
       source venv/bin/activate
 
-      sudo apt-get install libprotobuf-dev protobuf-compiler
-      python -m pip install -q --upgrade pip
-      python -m pip install -q -r requirements-release.txt
+      if [ '$(protobuf_type)' == 'External' ]; then
+        sudo apt-get install libprotobuf-dev protobuf-compiler
+      elif [ '$(protobuf_type)' == 'Internal' ]; then
+        echo "Use the internal protobuf build"
+      fi
+
+      python -m pip install --upgrade pip
+      python -m pip install -r requirements-release.txt
+      # still test protobuf==3.20.2 at least in a CI
+      python -m pip install protobuf==3.20.2
 
       sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y --no-install-recommends dos2unix
 
       git submodule update --init --recursive
       export ONNX_BUILD_TESTS=1
-      if [ '$(onnx_debug)' == '1' ]; then
+      if [ '$(build_type)' == 'Debug' ]; then
         export DEBUG=1
       fi
       if [ '$(onnx_ml)' == '1' ]; then
         export ONNX_ML=1
       fi
       export CMAKE_ARGS="-DONNX_WERROR=ON -DONNX_USE_PROTOBUF_SHARED_LIBS=ON"
-      export ONNX_NAMESPACE=ONNX_NAMESPACE_FOO_BAR_FOR_CI
-      python setup.py --quiet install
+      # Enable more sanitizers
+      export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CXX_FLAGS='-fsanitize=undefined -fno-sanitize-recover=all '"
+      pip install -e ".[reference]" -v
     displayName: 'Install ONNX and dependencies'
 
   - script: |
       source venv/bin/activate
 
-      # check line endings to be UNIX
-      find . -type f -regextype posix-extended -regex '.*\.(py|cpp|md|h|cc|proto|proto3|in)' | xargs dos2unix --quiet
-      git status
-      git diff --exit-code
-
-      # check auto-gen files up-to-date
-      python onnx/defs/gen_doc.py
-      python onnx/gen_proto.py -l
-      python onnx/gen_proto.py -l --ml
-      python onnx/backend/test/stat_coverage.py
-
-      git status
-      git diff --exit-code -- . ':(exclude)onnx/onnx-data.proto' ':(exclude)onnx/onnx-data.proto3'
-      if [ $? -ne 0 ]; then
-        echo "git diff returned failures"
-        exit 1
-      fi
-
-      # Do not hardcode onnx's namespace in the c++ source code, so that
-      # other libraries who statically link with onnx can hide onnx symbols
-      # in a private namespace.
-      ! grep -R --include='*.cc' --include='*.h' 'namespace onnx' .
-      ! grep -R --include='*.cc' --include='*.h' 'onnx::' .
-
-      pytest
+      pytest -sv --cov=onnx --cov-report=xml --cov-append --cov-branch --junit-xml pytest.xml -n auto --dist loadscope
       if [ $? -ne 0 ]; then
         echo "pytest failed"
         exit 1
@@ -107,6 +97,14 @@ jobs:
 
     displayName: 'Run ONNX tests'
 
+  - script: |
+      curl -Os https://uploader.codecov.io/latest/linux/codecov
+      chmod +x codecov
+      ./codecov
+
+    continueOnError: true
+    displayName: 'Upload to codecov'
+
   - script: |
       source venv/bin/activate
       python onnx/backend/test/cmd_tools.py generate-data --clean
@@ -124,13 +122,23 @@ jobs:
         exit 1
       fi
 
+      pip uninstall -y pillow
+      python onnx/backend/test/cmd_tools.py generate-data --clean
+      git status
+      # Verify test generation without pillow for ImageDecoder, it should directly use frozen data
+      git diff --exit-code -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' ':!*output_*.pb' ':!*input_*.pb'
+      if [ $? -ne 0 ]; then
+        echo "git diff for test generation without pillow returned failures. Please check updated node test files"
+        exit 1
+      fi
+
     displayName: Test backend test data
 
   - script: |
       if [ '$(documentation)' == '1' ]; then
         source venv/bin/activate
         pip install -r docs/docsgen/source/requirements.txt
-        cd docs/docsgen && make text -j auto
+        cd docs/docsgen && make text
       fi
     displayName: Test documentation
     continueOnError: true  # the documentation generates errors due to operators documentation
diff --git a/.azure-pipelines/MacOS-CI.yml b/.azure-pipelines/MacOS-CI.yml
@@ -1,38 +1,44 @@
+schedules:
+- cron: '0 0 * * *'
+  displayName: Nightly MacOS CI in main branch
+  branches:
+    include:
+    - main
+
 trigger:
-- main
+- gh-readonly-queue/**
 
 jobs:
 - job: 'Test'
   pool:
     vmImage: 'macOS-11'
   strategy:
     matrix:
-      py39:
-        python.version: '3.9'
+      py311-external-protobuf:
+        python.version: '3.11'
         onnx_ml: 0
-        onnx_debug: 0
+        build_type: 'Release'
+        protobuf_type: 'External'
         onnx_lite: 0
-      py38:
-        python.version: '3.8'
+      py310-lite-internal-protobuf:
+        python.version: '3.10'
         onnx_ml: 0
-        onnx_debug: 0
+        build_type: 'Release'
+        protobuf_type: 'Internal'
         onnx_lite: 1
-      py38-ml:
-        python.version: '3.8'
+      py39-ml-lite-external-protobuf:
+        python.version: '3.9'
         onnx_ml: 1
-        onnx_debug: 0
+        build_type: 'Release'
+        protobuf_type: 'External'
         onnx_lite: 1
-      py38-ml-debug:
+      py38-ml-debug-external-protobuf:
         python.version: '3.8'
         onnx_ml: 1
-        onnx_debug: 1
+        build_type: 'Debug'
+        protobuf_type: 'External'
         onnx_lite: 0
-      py37-ml-debug:
-        python.version: '3.7'
-        onnx_ml: 1
-        onnx_debug: 1
-        onnx_lite: 0
-    maxParallel: 6
+    maxParallel: 4
 
   steps:
   - task: UsePythonVersion@0
@@ -42,18 +48,18 @@ jobs:
   - script: |
       # Install Protobuf from source
       export NUM_CORES=`sysctl -n hw.logicalcpu`
-      if [ '$(onnx_debug)' == '1' ]; then
+      if [ '$(build_type)' == 'Debug' ]; then
         export DEBUG=1
-        source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install Debug
-      else
-        source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install
+      fi
+      if [ '$(protobuf_type)' == 'External' ]; then
+        source workflow_scripts/protobuf/build_protobuf_unix.sh $NUM_CORES $(pwd)/protobuf/protobuf_install $(build_type)
+      elif [ '$(protobuf_type)' == 'Internal' ]; then
+        echo "Use the internal protobuf build"
       fi
 
       git submodule update --init --recursive
-      python -m pip install -q --upgrade pip
-      python -m pip install -q -r requirements-release.txt
-      # Mac cannot work with Python Protobuf 3.18.3 https://github.com/protocolbuffers/protobuf/issues/10691
-      python -m pip install protobuf==3.19.5
+      python -m pip install --upgrade pip
+      python -m pip install -r requirements-release.txt
 
       if [ '$(onnx_ml)' == '1' ]; then
         export ONNX_ML=1
@@ -63,33 +69,23 @@ jobs:
       if [ '$(onnx_lite)' == '1' ]; then
         export CMAKE_ARGS="${CMAKE_ARGS} -DONNX_USE_LITE_PROTO=ON"
       fi
-      export ONNX_NAMESPACE=ONNX_NAMESPACE_FOO_BAR_FOR_CI
-      python setup.py -q install
+      pip install -e ".[reference]" -v
     displayName: 'Install dependencies and ONNX'
 
   - script: |
-      # check auto-gen files up-to-date
-      python onnx/defs/gen_doc.py
-      python onnx/gen_proto.py -l
-      python onnx/gen_proto.py -l --ml
-      python onnx/backend/test/stat_coverage.py
-
-      git status
-      git diff --exit-code  -- . ':(exclude)onnx/onnx-data.proto' ':(exclude)onnx/onnx-data.proto3'
+      pytest -n auto --dist loadscope
       if [ $? -ne 0 ]; then
-        echo "git diff returned failures"
+        echo "pytest failed"
         exit 1
       fi
 
-      # Do not hardcode onnx's namespace in the c++ source code, so that
-      # other libraries who statically link with onnx can hide onnx symbols
-      # in a private namespace.
-      ! grep -R --include='*.cc' --include='*.h' 'namespace onnx' .
-      ! grep -R --include='*.cc' --include='*.h' 'onnx::' .
-
-      pytest
+      python -m pip install onnxruntime
+      export ORT_MAX_IR_SUPPORTED_VERSION=8
+      export ORT_MAX_ML_OPSET_SUPPORTED_VERSION=3
+      export ORT_MAX_ONNX_OPSET_SUPPORTED_VERSION=19
+      pytest -n auto --dist loadscope
       if [ $? -ne 0 ]; then
-        echo "pytest failed"
+        echo "pytest failed when testing onnx with onnxruntime"
         exit 1
       fi
 
@@ -114,4 +110,19 @@ jobs:
         exit 1
       fi
 
+      pip uninstall -y pillow
+      python onnx/backend/test/cmd_tools.py generate-data --clean
+      git status
+      git diff --exit-code -- . ':!onnx/onnx-data.proto' ':!onnx/onnx-data.proto3' ':!*output_*.pb' ':!*input_*.pb'
+      if [ $? -ne 0 ]; then
+        echo "git diff for test generation without pillow returned failures. Please check updated node test files"
+        exit 1
+      fi
+      # Internal Protobuf won't have other untrack files like protobuf/
+      if [ '$(protobuf_type)' == 'Internal' ]; then
+        if [[ $(git ls-files --others --exclude-standard) ]]; then
+          echo "Some test-generated files not included in the PR. Did you forget to add any test-generated files?"
+          exit 1
+        fi
+      fi
     displayName: 'Run ONNX Tests'