Upgrade to newer Protobuf 3.20.2 (#4629)

* Upgrade to newer Protobuf 3.20.2 Signed-off-by: Chun-Wei Chen <jacky82226@gmail.com> * minimum 3.20.2 Signed-off-by: Chun-Wei Chen <jacky82226@gmail.com> * fix Signed-off-by: Chun-Wei Chen <jacky82226@gmail.com> * revert uninented Signed-off-by: Chun-Wei Chen <jacky82226@gmail.com> Signed-off-by: Chun-Wei Chen <jacky82226@gmail.com> Co-authored-by: G. Ramalingam <grama@microsoft.com>
onnx · Nov 14, 2022 · d006f33 · d006f33
1 parent 70c4a7d
commit d006f33
Show file tree

Hide file tree

Showing 13 changed files with 20 additions and 28 deletions.
diff --git a/.azure-pipelines/MacOS-CI.yml b/.azure-pipelines/MacOS-CI.yml
@@ -52,8 +52,6 @@ jobs:
       git submodule update --init --recursive
       python -m pip install -q --upgrade pip
       python -m pip install -q -r requirements-release.txt
-      # Mac cannot work with Python Protobuf 3.18.3 https://github.com/protocolbuffers/protobuf/issues/10691
-      python -m pip install protobuf==3.19.5
 
       if [ '$(onnx_ml)' == '1' ]; then
         export ONNX_ML=1

diff --git a/.azure-pipelines/Windows-CI.yml b/.azure-pipelines/Windows-CI.yml
@@ -41,7 +41,7 @@ jobs:
 
   - script: |
       conda create --yes --quiet --name py$(python.version) python=$(python.version)
-      conda install -n py$(python.version) -y -c conda-forge numpy libprotobuf=3.18
+      conda install -n py$(python.version) -y -c conda-forge numpy libprotobuf=3.20
     displayName: Create Anaconda environment
 
   - script: |

diff --git a/.github/workflows/release_linux_aarch64.yml b/.github/workflows/release_linux_aarch64.yml
@@ -106,7 +106,7 @@ jobs:
           ${{ env.img }} \
           bash -exc '\
           source .env/bin/activate && \
-          python -m pip uninstall -y onnx && python -m pip install protobuf==3.12.2 && \
+          python -m pip uninstall -y onnx && python -m pip install protobuf==3.20.2 && \
           python -m pip install dist/*manylinux2014_aarch64.whl && \
           pytest && \
           deactivate'

diff --git a/.github/workflows/release_linux_x86_64.yml b/.github/workflows/release_linux_x86_64.yml
@@ -82,7 +82,7 @@ jobs:
     - name: Verify ONNX with the minimum supported protobuf (from requirements.txt)
       if: ${{ always() }}
       run: |
-        python -m pip uninstall -y protobuf onnx && python -m pip install protobuf==3.12.2
+        python -m pip uninstall -y protobuf onnx && python -m pip install protobuf==3.20.2
         python -m pip install dist/*manylinux2014_x86_64.whl
         pytest
 

diff --git a/.github/workflows/release_mac.yml b/.github/workflows/release_mac.yml
@@ -42,8 +42,6 @@ jobs:
       run: |
         python -m pip install -q --upgrade pip
         python -m pip install -q -r requirements-release.txt
-        # Mac cannot work with Python Protobuf 3.18.3 https://github.com/protocolbuffers/protobuf/issues/10691
-        python -m pip install protobuf==3.19.5
 
     - name: Build wheel and install
       env:
@@ -100,7 +98,7 @@ jobs:
     - name: Verify ONNX with the minimum supported protobuf (from requirements.txt)
       if: ${{ always() }}
       run: |
-        python -m pip uninstall -y protobuf onnx && python -m pip install protobuf==3.12.2
+        python -m pip uninstall -y protobuf onnx && python -m pip install protobuf==3.20.2
         for file in dist/*.whl; do python -m pip install --upgrade $file; done
         pytest
 

diff --git a/.github/workflows/release_win.yml b/.github/workflows/release_win.yml
@@ -114,7 +114,7 @@ jobs:
       if: ${{ always() }}
       run: |
         cd onnx
-        python -m pip uninstall -y protobuf onnx && python -m pip install protobuf==3.12.2
+        python -m pip uninstall -y protobuf onnx && python -m pip install protobuf==3.20.2
         Get-ChildItem -Path dist/*.whl | foreach {python -m pip install --upgrade $_.fullname}
         pytest
 

diff --git a/.github/workflows/weekly_mac_ci.yml b/.github/workflows/weekly_mac_ci.yml
@@ -34,8 +34,6 @@ jobs:
         set -e
         python -m pip install -q --upgrade pip
         python -m pip install -q -r requirements-release.txt
-        # Mac cannot work with Python Protobuf 3.18.3 https://github.com/protocolbuffers/protobuf/issues/10691
-        python -m pip install protobuf==3.19.5
 
     - name: Build, install and test ONNX
       shell: bash
@@ -47,8 +45,6 @@ jobs:
         export CC=clang
         export CXX=clang++
         export ONNX_ML=1
-        export ONNX_NAMESPACE=ONNX_REL_1_8
-        export MACOSX_DEPLOYMENT_TARGET=10.15
         python setup.py --quiet install
   
     - name: Test all models with onnx.checker and onnx.shape_inference

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -11,4 +11,4 @@
         "--profile",
         "black"
     ],
-}
+}
diff --git a/README.md b/README.md
@@ -94,15 +94,15 @@ The ON/OFF depends on what kind of protobuf library you have. Shared libraries a
 
 
 ### Windows
-If you are building ONNX from source, it is recommended that you also build Protobuf locally as a static library. The version distributed with conda-forge is a DLL, but ONNX expects it to be a static library. Building protobuf locally also lets you control the version of protobuf. The tested and recommended version is 3.18.3.
+If you are building ONNX from source, it is recommended that you also build Protobuf locally as a static library. The version distributed with conda-forge is a DLL, but ONNX expects it to be a static library. Building protobuf locally also lets you control the version of protobuf. The tested and recommended version is 3.20.2.
 
 The instructions in this README assume you are using Visual Studio.  It is recommended that you run all the commands from a shell started from "x64 Native Tools Command Prompt for VS 2019" and keep the build system generator for cmake (e.g., cmake -G "Visual Studio 16 2019") consistent while building protobuf as well as ONNX.
 
 You can get protobuf by running the following commands:
 ```bat
 git clone https://github.com/protocolbuffers/protobuf.git
 cd protobuf
-git checkout v3.18.3
+git checkout v3.20.2
 cd cmake
 cmake -G "Visual Studio 16 2019" -A x64 -DCMAKE_INSTALL_PREFIX=<protobuf_install_dir> -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_EXAMPLES=OFF .
 msbuild protobuf.sln /m /p:Configuration=Release
@@ -150,7 +150,7 @@ A more general way is to build and install it from source. See the instructions
   ```bash
     git clone https://github.com/protocolbuffers/protobuf.git
     cd protobuf
-    git checkout v3.18.3
+    git checkout v3.20.2
     git submodule update --init --recursive
     mkdir build_source && cd build_source
     cmake ../cmake -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
@@ -162,7 +162,7 @@ A more general way is to build and install it from source. See the instructions
   ```bash
     git clone https://github.com/protocolbuffers/protobuf.git
     cd protobuf
-    git checkout v3.18.3
+    git checkout v3.20.2
     git submodule update --init --recursive
     mkdir build_source && cd build_source
     cmake ../cmake  -DCMAKE_INSTALL_LIBDIR=lib64 -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
@@ -193,9 +193,9 @@ pip install -e .
 export NUM_CORES=`sysctl -n hw.ncpu`
 brew update
 brew install autoconf && brew install automake
-wget https://github.com/protocolbuffers/protobuf/releases/download/v3.18.3/protobuf-cpp-3.18.3.tar.gz
-tar -xvf protobuf-cpp-3.18.3.tar.gz
-cd protobuf-3.18.3
+wget https://github.com/protocolbuffers/protobuf/releases/download/v3.20.2/protobuf-cpp-3.20.2.tar.gz
+tar -xvf protobuf-cpp-3.20.2.tar.gz
+cd protobuf-3.20.2
 mkdir build_source && cd build_source
 cmake ../cmake -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
 make -j${NUM_CORES}

diff --git a/requirements-release.txt b/requirements-release.txt
@@ -1,5 +1,5 @@
 numpy == 1.21.5
-protobuf == 3.18.3
+protobuf == 3.20.2
 pytest
 nbval
 ipython

diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
 numpy >= 1.16.6 # TODO: update once TensorFlow 2.6 is end of life
-protobuf >= 3.12.2, < 4
+protobuf >= 3.20.2, < 4
 typing-extensions >= 3.6.2.1
diff --git a/workflow_scripts/protobuf/build_protobuf_unix.sh b/workflow_scripts/protobuf/build_protobuf_unix.sh
@@ -21,9 +21,9 @@ fi
 # Build protobuf from source with -fPIC on Unix-like system
 ORIGINAL_PATH=$(pwd)
 cd ..
-wget https://github.com/protocolbuffers/protobuf/releases/download/v3.18.3/protobuf-cpp-3.18.3.tar.gz
-tar -xvf protobuf-cpp-3.18.3.tar.gz
-cd protobuf-3.18.3
+wget https://github.com/protocolbuffers/protobuf/releases/download/v3.20.2/protobuf-cpp-3.20.2.tar.gz
+tar -xvf protobuf-cpp-3.20.2.tar.gz
+cd protobuf-3.20.2
 mkdir build_source && cd build_source
 cmake ../cmake -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=$INSTALL_PROTOBUF_PATH -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$BUILD_TYPE
 make -j$CORE_NUMBER

diff --git a/workflow_scripts/protobuf/build_protobuf_win.ps1 b/workflow_scripts/protobuf/build_protobuf_win.ps1
@@ -6,9 +6,9 @@ param(
 )
 
 echo "Build protobuf from source on Windows."
-Invoke-WebRequest -Uri https://github.com/protocolbuffers/protobuf/releases/download/v3.18.3/protobuf-cpp-3.18.3.tar.gz -OutFile protobuf.tar.gz -Verbose
+Invoke-WebRequest -Uri https://github.com/protocolbuffers/protobuf/releases/download/v3.20.2/protobuf-cpp-3.20.2.tar.gz -OutFile protobuf.tar.gz -Verbose
 tar -xvf protobuf.tar.gz
-cd protobuf-3.18.3
+cd protobuf-3.20.2
 $protobuf_root_dir = Get-Location
 mkdir protobuf_install
 cd cmake