Changes for PyTorch 2.1.0 and Python 3.11

.github/workflows/ci_gpu.yml

@@ -39,7 +39,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Install dependencies
         run: |
-          python ts_scripts/install_dependencies.py --environment=dev --cuda=cu117
+          python ts_scripts/install_dependencies.py --environment=dev --cuda=cu121
       - name: Torchserve Sanity
         uses: nick-fields/retry@v2
         with:

.github/workflows/regression_tests_gpu.yml

@@ -43,7 +43,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Install dependencies
         run: |
-          python ts_scripts/install_dependencies.py --environment=dev --cuda=cu117
+          python ts_scripts/install_dependencies.py --environment=dev --cuda=cu121
       - name: Torchserve Regression Tests
         run: |
           python test/regression_tests.py

CONTRIBUTING.md

@@ -20,9 +20,9 @@ Your contributions will fall into two categories:
 
          For GPU
            ```bash
-           python ts_scripts/install_dependencies.py --environment=dev --cuda=cu118
+           python ts_scripts/install_dependencies.py --environment=dev --cuda=cu121
            ```
-            > Supported cuda versions as cu118, cu117, cu116, cu113, cu111, cu102, cu101, cu92
+            > Supported cuda versions as cu121, cu118, cu117, cu116, cu113, cu111, cu102, cu101, cu92
         - Install `pre-commit` to your Git flow:
             ```bash
             pre-commit install

README.md

@@ -17,7 +17,7 @@ curl http://127.0.0.1:8080/predictions/bert -T input.txt
 ```
 # Install dependencies
 # cuda is optional
-python ./ts_scripts/install_dependencies.py --cuda=cu102
+python ./ts_scripts/install_dependencies.py --cuda=cu121
 
 # Latest release
 pip install torchserve torch-model-archiver torch-workflow-archiver
@@ -31,7 +31,7 @@ pip install torchserve-nightly torch-model-archiver-nightly torch-workflow-archi
 ```
 # Install dependencies
 # cuda is optional
-python ./ts_scripts/install_dependencies.py --cuda=cu102
+python ./ts_scripts/install_dependencies.py --cuda=cu121
 
 # Latest release
 conda install -c pytorch torchserve torch-model-archiver torch-workflow-archiver

benchmarks/auto_benchmark.py

@@ -149,7 +149,7 @@ def install_torchserve(skip_ts_install, hw, ts_version):
 
     # install_dependencies.py
     if hw == "gpu":
-        cmd = "python ts_scripts/install_dependencies.py --environment dev --cuda cu118"
+        cmd = "python ts_scripts/install_dependencies.py --environment dev --cuda cu121"
     elif hw == "neuronx":
         cmd = "python ts_scripts/install_dependencies.py --environment dev --neuronx"
     else:

binaries/conda/build_packages.py

@@ -130,7 +130,7 @@ def conda_build(
 
     os.environ["PYTHON"] = "python"
 
-    python_versions = ["3.8", "3.9", "3.10"]
+    python_versions = ["3.8", "3.9", "3.10", "3.11"]
     packages = [os.path.join(conda_build_dir, pkg) for pkg in PACKAGES]
 
     # Generate conda binaries for linux-64

docker/README.md

@@ -37,11 +37,11 @@ Use `build_image.sh` script to build the docker images. The script builds the `p
 |-bi, --baseimage specify base docker image. Example: nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu20.04|
 |-bt, --buildtype|Which type of docker image to build. Can be one of : production, dev, ci, codebuild|
 |-t, --tag|Tag name for image. If not specified, script uses torchserve default tag names.|
-|-cv, --cudaversion| Specify to cuda version to use. Supported values `cu92`, `cu101`, `cu102`, `cu111`, `cu113`, `cu116`, `cu117`, `cu118`. Default `cu117`|
+|-cv, --cudaversion| Specify to cuda version to use. Supported values `cu92`, `cu101`, `cu102`, `cu111`, `cu113`, `cu116`, `cu117`, `cu118`. `cu121`, Default `cu121`|
 |-ipex, --build-with-ipex| Specify to build with intel_extension_for_pytorch. If not specified, script builds without intel_extension_for_pytorch.|
 |-n, --nightly| Specify to build with TorchServe nightly.|
 |--codebuild| Set if you need [AWS CodeBuild](https://aws.amazon.com/codebuild/)|
-|-py, --pythonversion| Specify the python version to use. Supported values `3.8`, `3.9`, `3.10`. Default `3.9`|
+|-py, --pythonversion| Specify the python version to use. Supported values `3.8`, `3.9`, `3.10`, `3.11`. Default `3.9`|
 
 
 **PRODUCTION ENVIRONMENT IMAGES**

docker/build_image.sh

@@ -82,10 +82,10 @@ do
           ;;
         -py|--pythonversion)
           PYTHON_VERSION="$2"
-          if [[ $PYTHON_VERSION = 3.8 || $PYTHON_VERSION = 3.9 || $PYTHON_VERSION = 3.10 ]]; then
+          if [[ $PYTHON_VERSION = 3.8 || $PYTHON_VERSION = 3.9 || $PYTHON_VERSION = 3.10 || $PYTHON_VERSION = 3.11 ]]; then
             echo "Valid python version"
           else
-            echo "Valid python versions are 3.8, 3.9 and 3.10"
+            echo "Valid python versions are 3.8, 3.9 3.10 and 3.11"
             exit 1
           fi
           shift
@@ -94,7 +94,10 @@ do
         # With default ubuntu version 20.04
         -cv|--cudaversion)
           CUDA_VERSION="$2"
-          if [ "${CUDA_VERSION}" == "cu118" ];
+          if [ "${CUDA_VERSION}" == "cu121" ];
+          then
+            BASE_IMAGE="nvidia/cuda:12.1.0-base-ubuntu20.04"
+          elif [ "${CUDA_VERSION}" == "cu118" ];
           then
             BASE_IMAGE="nvidia/cuda:11.8.0-base-ubuntu20.04"
           elif [ "${CUDA_VERSION}" == "cu117" ];

docker/build_upload_release.py

@@ -28,7 +28,7 @@
     # Upload pytorch/torchserve docker binaries
     try_and_handle(f"./build_image.sh -t {organization}/torchserve:latest", dry_run)
     try_and_handle(
-        f"./build_image.sh -g -cv cu118 -t {organization}/torchserve:latest-gpu",
+        f"./build_image.sh -g -cv cu121 -t {organization}/torchserve:latest-gpu",
         dry_run,
     )
     try_and_handle(

docker/docker_nightly.py

@@ -39,7 +39,7 @@
     # Build Nightly images and append the date in the name
     try_and_handle(f"./build_image.sh -n -t {organization}/{cpu_version}", dry_run)
     try_and_handle(
-        f"./build_image.sh -g -cv cu118 -t {organization}/{gpu_version}",
+        f"./build_image.sh -g -cv cu121 -t {organization}/{gpu_version}",
         dry_run,
     )
 

docs/code_coverage.md

@@ -10,9 +10,9 @@
    For GPU
     - Install dependencies (if not already installed)
    ```bash
-   python ts_scripts/install_dependencies.py --environment=dev --cuda=cu102
+   python ts_scripts/install_dependencies.py --environment=dev --cuda=cu121
    ```
-   > Supported cuda versions as cu118, cu117, cu116, cu113, cu111, cu102, cu101, cu92
+   > Supported cuda versions as cu121, cu118, cu117, cu116, cu113, cu111, cu102, cu101, cu92
 
  - Execute sanity suite
    ```bash

docs/getting_started.md

@@ -14,10 +14,10 @@
         python ./ts_scripts/install_dependencies.py
         ```
 
-     - For GPU with Cuda 10.2. Options are `cu92`, `cu101`, `cu102`, `cu111`, `cu113`, `cu116`, `cu117`, `cu118`
+     - For GPU with Cuda 12.1. Options are `cu92`, `cu101`, `cu102`, `cu111`, `cu113`, `cu116`, `cu117`, `cu118`, `cu121`
 
        ```bash
-       python ./ts_scripts/install_dependencies.py --cuda=cu102
+       python ./ts_scripts/install_dependencies.py --cuda=cu121
        ```
 
      Note: PyTorch 1.9+ will not support cu92 and cu101. So TorchServe only supports cu92 and cu101 up to PyTorch 1.8.1.

requirements/torch_cu121_linux.txt

@@ -0,0 +1,7 @@
+#pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu121
+--extra-index-url https://download.pytorch.org/whl/test/cu121
+-r torch_common.txt
+torch==2.1.0+cu121; sys_platform == 'linux'
+torchvision==0.16.0+cu121; sys_platform == 'linux'
+torchtext==0.16.0; sys_platform == 'linux'
+torchaudio==2.1.0+cu121; sys_platform == 'linux'

requirements/torch_cu121_windows.txt

@@ -0,0 +1,7 @@
+#pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu121
+--extra-index-url https://download.pytorch.org/whl/cu121
+-r torch_common.txt
+torch==2.1.0+cu121; sys_platform == 'linux'
+torchvision==0.16.0+cu121; sys_platform == 'linux'
+torchtext==0.16.0; sys_platform == 'linux'
+torchaudio==2.1.0+cu121; sys_platform == 'linux'

requirements/torch_darwin.txt

@@ -1,6 +1,7 @@
 #pip install torch torchvision torchaudio
+--extra-index-url https://download.pytorch.org/whl/test/cpu
 -r torch_common.txt
-torch==2.0.1; sys_platform == 'darwin'
-torchvision==0.15.2; sys_platform == 'darwin'
-torchtext==0.15.2; sys_platform == 'darwin'
-torchaudio==2.0.2; sys_platform == 'darwin'
+torch==2.1.0; sys_platform == 'darwin'
+torchvision==0.16.0; sys_platform == 'darwin'
+torchtext==0.16.0; sys_platform == 'darwin'
+torchaudio==2.1.0; sys_platform == 'darwin'

requirements/torch_linux.txt

@@ -1,7 +1,7 @@
 #pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
---extra-index-url https://download.pytorch.org/whl/cpu
+--extra-index-url https://download.pytorch.org/whl/test/cpu
 -r torch_common.txt
-torch==2.0.1+cpu; sys_platform == 'linux'
-torchvision==0.15.2+cpu; sys_platform == 'linux'
-torchtext==0.15.2; sys_platform == 'linux'
-torchaudio==2.0.2+cpu; sys_platform == 'linux'
+torch==2.1.0+cpu; sys_platform == 'linux'
+torchvision==0.16.0+cpu; sys_platform == 'linux'
+torchtext==0.16.0; sys_platform == 'linux'
+torchaudio==2.1.0+cpu; sys_platform == 'linux'

requirements/torch_windows.txt

@@ -1,6 +1,7 @@
 #pip install torch torchvision torchaudio
+--extra-index-url https://download.pytorch.org/whl/test/cpu
 -r torch_common.txt
-torch==2.0.1; sys_platform == 'win32'
-torchvision==0.15.2; sys_platform == 'win32'
-torchtext==0.15.2; sys_platform == 'win32'
-torchaudio==2.0.2; sys_platform == 'win32'
+torch==2.1.0; sys_platform == 'win32'
+torchvision==0.16.0; sys_platform == 'win32'
+torchtext==0.16.0; sys_platform == 'win32'
+torchaudio==2.1.0; sys_platform == 'win32'

ts_scripts/install_dependencies.py

@@ -204,7 +204,17 @@ def get_brew_version():
     parser.add_argument(
         "--cuda",
         default=None,
-        choices=["cu92", "cu101", "cu102", "cu111", "cu113", "cu116", "cu117", "cu118"],
+        choices=[
+            "cu92",
+            "cu101",
+            "cu102",
+            "cu111",
+            "cu113",
+            "cu116",
+            "cu117",
+            "cu118",
+            "cu121",
+        ],
         help="CUDA version for torch",
     )
     parser.add_argument(