manywheel: Add support for Python 3.9

conda/build_pytorch.sh

@@ -320,13 +320,21 @@ for py_ver in "${DESIRED_PYTHON[@]}"; do
     # Build the package
     echo "Build $build_folder for Python version $py_ver"
     conda config --set anaconda_upload no
+
+    ADDITIONAL_CHANNELS=""
+    # TODO: Remove ADDITIONAL_CHANNELS
+    # Nov, 2020: There's an issue with installing python 3.9 directly from
+    #            anaconda so we need to use conda-forge
+    if [[ ${py_ver} = "3.9" ]]; then
+      ADDITIONAL_CHANNELS="-c=conda-forge"
+    fi
     echo "Calling conda-build at $(date)"
     time CMAKE_ARGS=${CMAKE_ARGS[@]} \
          EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \
          PYTORCH_GITHUB_ROOT_DIR="$pytorch_rootdir" \
          PYTORCH_BUILD_STRING="$build_string" \
          PYTORCH_MAGMA_CUDA_VERSION="$cuda_nodot" \
-         conda build -c "$ANACONDA_USER" \
+         conda build -c "$ANACONDA_USER" ${ADDITIONAL_CHANNELS} \
                      --no-anaconda-upload \
                      --python "$py_ver" \
                      --output-folder "$output_folder" \

conda/pytorch-nightly/meta.yaml

@@ -12,7 +12,7 @@ requirements:
 
   host:
     - python
-    - numpy 1.11.*
+    - numpy >=1.11.*
     - setuptools
     - pyyaml
     - mkl >=2019

manywheel/build_common.sh

@@ -65,14 +65,16 @@ export CMAKE_INCLUDE_PATH="/opt/intel/include:$CMAKE_INCLUDE_PATH"
 # ever pass one python version, so we assume that DESIRED_PYTHON is not a list
 # in this case
 if [[ -n "$DESIRED_PYTHON" && "$DESIRED_PYTHON" != cp* ]]; then
-    if [[ "$DESIRED_PYTHON" == '2.7mu' ]]; then
-      DESIRED_PYTHON='cp27-cp27mu'
-    elif [[ "$DESIRED_PYTHON" == '3.8m' ]]; then
-      DESIRED_PYTHON='cp38-cp38'
-    else
-      python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)"
-      DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}m"
-    fi
+    python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)"
+    case ${DESIRED_PYTHON} in
+      3.[6-7])
+        DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}m"
+        ;;
+      # Should catch 3.8+
+      3.*)
+        DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}"
+        ;;
+    esac
 fi
 py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:1}"
 pydir="/opt/python/$DESIRED_PYTHON"
@@ -108,13 +110,18 @@ fi
 #######################################################
 python setup.py clean
 retry pip install -qr requirements.txt
-if [[ "$DESIRED_PYTHON"  == "cp37-cp37m" ]]; then
-    retry pip install -q numpy==1.15
-elif [[ "$DESIRED_PYTHON"  == "cp38-cp38" ]]; then
-    retry pip install -q numpy==1.15
-else
+case ${DESIRED_PYTHON} in
+  cp36-cp36m)
     retry pip install -q numpy==1.11
-fi
+    ;;
+  cp3[7-8]*)
+    retry pip install -q numpy==1.15
+    ;;
+  # Should catch 3.9+
+  *)
+    retry pip install -q numpy==1.19.4
+    ;;
+esac
 
 if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then
     export _GLIBCXX_USE_CXX11_ABI=1

manywheel/build_scripts/build.sh

@@ -5,7 +5,7 @@
 set -ex
 
 # Python versions to be installed in /opt/$VERSION_NO
-CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"2.7.15 3.5.5 3.6.6 3.7.5 3.8.1"}
+CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"3.6.6 3.7.5 3.8.1 3.9.0"}
 
 # openssl version to build, with expected sha256 hash of .tar.gz
 # archive

run_tests.sh

@@ -75,7 +75,9 @@ if [[ "$package_type" == conda || "$(uname)" == Darwin ]]; then
             retry conda install -yq cpuonly -c pytorch
         fi
     fi
-    if [[ "$(python --version 2>&1)" == *3.8.* ]]; then
+    if [[ "$(python --version 2>&1)" == *3.9.* ]]; then
+        retry conda install -yq -c conda-forge future hypothesis mkl>=2018 ninja numpy>=1.15 protobuf pytest setuptools six typing_extensions pyyaml
+    elif [[ "$(python --version 2>&1)" == *3.8.* ]]; then
         retry conda install -yq future hypothesis mkl>=2018 ninja numpy>=1.15 protobuf pytest setuptools six typing_extensions pyyaml
     elif [[ "$(python --version 2>&1)" == *3.6.* ]]; then
         retry conda install -yq cffi future hypothesis mkl>=2018 ninja numpy>=1.11 protobuf pytest setuptools six typing_extensions pyyaml requests dataclasses

smoke_test.sh

@@ -94,11 +94,18 @@ fi
 if [[ "$PACKAGE_TYPE" == 'conda' || "$(uname)" == 'Darwin' ]]; then
   # Create a new conda env in conda, or on MacOS
   conda create -yn test python="$py_dot" && source activate test
-  if [[ "$(python --version 2>&1)" == *3.6.* ]]; then
-    retry conda install -yq future numpy protobuf six requests dataclasses
-  else
-    retry conda install -yq future numpy protobuf six requests
-  fi
+  python_version=$(python --version 2>&1)
+  dependencies="numpy protobuf six requests"
+  case ${python_version} in
+    *3.6.*)
+      dependencies="${dependencies} future dataclasses"
+      ;;
+    # TODO: Remove this once Python 3.9 is workable through the default conda channels
+    *3.9.*)
+      dependencies="-c=conda-forge ${dependencies}"
+      ;;
+  esac
+  conda install -yq ${dependencies}
 else
   export PATH=/opt/python/${py_long}/bin:$PATH
   if [[ "$(python --version 2>&1)" == *3.6.* ]]; then

wheel/build_wheel.sh

@@ -109,10 +109,6 @@ fi
 wheel_filename_new="${TORCH_PACKAGE_NAME}-${build_version}${build_number_prefix}-cp${python_nodot}-none-${mac_version}.whl"
 
 ###########################################################
-# Install into a fresh env
-tmp_env_name="wheel_py$python_nodot"
-conda create -yn "$tmp_env_name" python="$desired_python"
-source activate "$tmp_env_name"
 
 # Have a separate Pytorch repo clone
 if [[ ! -d "$pytorch_rootdir" ]]; then
@@ -137,30 +133,36 @@ export INSTALL_TEST=0 # dont install test binaries into site-packages
 export MACOSX_DEPLOYMENT_TARGET=10.10
 export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
 
-SETUPTOOLS_PINNED_VERSION=""
-PYYAML_PINNED_VERSION=""
+SETUPTOOLS_PINNED_VERSION="=46.0.0"
+PYYAML_PINNED_VERSION="=5.3"
+EXTRA_CONDA_INSTALL_FLAGS=""
 case ${desired_python} in
-    3.5 | 2.7)
-        # setuptools and pyyaml discontinued support for python 3.5 and 2.7
-        continue
+    3.9)
+        EXTRA_CONDA_INSTALL_FLAGS="-c=conda-forge"
+        SETUPTOOLS_PINNED_VERSION=">=46.0.0"
+        PYYAML_PINNED_VERSION=">=5.3"
+        NUMPY_PINNED_VERSION=">=1.19"
         ;;
-    3*)
-        SETUPTOOLS_PINNED_VERSION="=46.0.0"
-        PYYAML_PINNED_VERSION="=5.3"
+    3.8)
+        NUMPY_PINNED_VERSION="=1.17"
+        ;;
+    *)
+        NUMPY_PINNED_VERSION="=1.11.3"
         ;;
 esac
 
-if [[ "$desired_python" == 3.8 ]]; then
-    retry conda install -yq cmake numpy=1.17 nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" ninja
-else
-    retry conda install -yq cmake numpy==1.11.3 nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" cffi typing_extensions ninja requests
-fi
-retry conda install -yq mkl-include==2020.1 mkl-static==2020.1 -c intel
+# Install into a fresh env
+tmp_env_name="wheel_py$python_nodot"
+conda create ${EXTRA_CONDA_INSTALL_FLAGS} -yn "$tmp_env_name" python="$desired_python"
+source activate "$tmp_env_name"
+
+retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq cmake "numpy${NUMPY_PINNED_VERSION}" nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" cffi typing_extensions ninja requests
+retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq mkl-include==2020.1 mkl-static==2020.1 -c intel
 retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true
 
 # For USE_DISTRIBUTED=1 on macOS, need libuv and pkg-config to find libuv.
 export USE_DISTRIBUTED=1
-retry conda install -yq libuv pkg-config
+retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq libuv pkg-config
 
 pushd "$pytorch_rootdir"
 echo "Calling setup.py bdist_wheel at $(date)"

windows/build_pytorch.bat

@@ -59,6 +59,7 @@ FOR %%v IN (%DESIRED_PYTHON%) DO (
     if "%%v" == "3.6" conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl>=2019" cffi pyyaml boto3 cmake ninja typing_extensions dataclasses python=%%v
     if "%%v" == "3.7" conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl>=2019" cffi pyyaml boto3 cmake ninja typing_extensions python=%%v
     if "%%v" == "3.8" conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl>=2019" pyyaml boto3 cmake ninja typing_extensions python=%%v
+    if "%%v" == "3.9" conda create -n py!PYTHON_VERSION_STR! -y -q -c conda-forge "numpy>=1.11" "mkl>=2019" pyyaml boto3 cmake ninja typing_extensions python=%%v
     if "%%v" == "3" conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl>=2019" pyyaml boto3 cmake ninja typing_extensions python=%%v
 )
 endlocal