From 51761805f15a22f8af1498a491fcfeea7cbe397e Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Mon, 2 Nov 2020 14:29:24 -0800 Subject: [PATCH 1/7] manywheel: Add support for Python 3.9 Signed-off-by: Eli Uriegas --- conda/build_pytorch.sh | 10 ++++++++- conda/pytorch-nightly/meta.yaml | 2 +- manywheel/build_common.sh | 35 +++++++++++++++++++------------- manywheel/build_scripts/build.sh | 2 +- wheel/build_wheel.sh | 25 +++++++++++------------ windows/build_pytorch.bat | 1 + 6 files changed, 45 insertions(+), 30 deletions(-) diff --git a/conda/build_pytorch.sh b/conda/build_pytorch.sh index c68b28082..cd7bf0019 100755 --- a/conda/build_pytorch.sh +++ b/conda/build_pytorch.sh @@ -320,13 +320,21 @@ for py_ver in "${DESIRED_PYTHON[@]}"; do # Build the package echo "Build $build_folder for Python version $py_ver" conda config --set anaconda_upload no + + ADDITIONAL_CHANNELS="" + # TODO: Remove ADDITIONAL_CHANNELS + # Nov, 2020: There's an issue with installing python 3.9 directly from + # anaconda so we need to use conda-forge + if [[ ${py_ver} = "3.9" ]]; then + ADDITIONAL_CHANNELS="-c=conda-forge" + fi echo "Calling conda-build at $(date)" time CMAKE_ARGS=${CMAKE_ARGS[@]} \ EXTRA_CAFFE2_CMAKE_FLAGS=${EXTRA_CAFFE2_CMAKE_FLAGS[@]} \ PYTORCH_GITHUB_ROOT_DIR="$pytorch_rootdir" \ PYTORCH_BUILD_STRING="$build_string" \ PYTORCH_MAGMA_CUDA_VERSION="$cuda_nodot" \ - conda build -c "$ANACONDA_USER" \ + conda build -c "$ANACONDA_USER" ${ADDITIONAL_CHANNELS} \ --no-anaconda-upload \ --python "$py_ver" \ --output-folder "$output_folder" \ diff --git a/conda/pytorch-nightly/meta.yaml b/conda/pytorch-nightly/meta.yaml index 79a9ac8a5..1f919857a 100644 --- a/conda/pytorch-nightly/meta.yaml +++ b/conda/pytorch-nightly/meta.yaml @@ -12,7 +12,7 @@ requirements: host: - python - - numpy 1.11.* + - numpy >=1.11.* - setuptools - pyyaml - mkl >=2019 diff --git a/manywheel/build_common.sh b/manywheel/build_common.sh index 2facb0e58..9720bc943 100644 --- a/manywheel/build_common.sh +++ b/manywheel/build_common.sh @@ -65,14 +65,16 @@ export CMAKE_INCLUDE_PATH="/opt/intel/include:$CMAKE_INCLUDE_PATH" # ever pass one python version, so we assume that DESIRED_PYTHON is not a list # in this case if [[ -n "$DESIRED_PYTHON" && "$DESIRED_PYTHON" != cp* ]]; then - if [[ "$DESIRED_PYTHON" == '2.7mu' ]]; then - DESIRED_PYTHON='cp27-cp27mu' - elif [[ "$DESIRED_PYTHON" == '3.8m' ]]; then - DESIRED_PYTHON='cp38-cp38' - else - python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)" - DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}m" - fi + python_nodot="$(echo $DESIRED_PYTHON | tr -d m.u)" + case ${DESIRED_PYTHON} in + 3.[6-7]) + DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}m" + ;; + # Should catch 3.8+ + 3.*) + DESIRED_PYTHON="cp${python_nodot}-cp${python_nodot}" + ;; + esac fi py_majmin="${DESIRED_PYTHON:2:1}.${DESIRED_PYTHON:3:1}" pydir="/opt/python/$DESIRED_PYTHON" @@ -108,13 +110,18 @@ fi ####################################################### python setup.py clean retry pip install -qr requirements.txt -if [[ "$DESIRED_PYTHON" == "cp37-cp37m" ]]; then - retry pip install -q numpy==1.15 -elif [[ "$DESIRED_PYTHON" == "cp38-cp38" ]]; then - retry pip install -q numpy==1.15 -else +case ${DESIRED_PYTHON} in + cp36-cp36m) retry pip install -q numpy==1.11 -fi + ;; + cp3[7-8]*) + retry pip install -q numpy==1.15 + ;; + # Should catch 3.9+ + *) + retry pip install -q numpy==1.19.4 + ;; +esac if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then export _GLIBCXX_USE_CXX11_ABI=1 diff --git a/manywheel/build_scripts/build.sh b/manywheel/build_scripts/build.sh index 666b972b5..dd5b28d22 100644 --- a/manywheel/build_scripts/build.sh +++ b/manywheel/build_scripts/build.sh @@ -5,7 +5,7 @@ set -ex # Python versions to be installed in /opt/$VERSION_NO -CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"2.7.15 3.5.5 3.6.6 3.7.5 3.8.1"} +CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"2.7.15 3.5.5 3.6.6 3.7.5 3.8.1 3.9.0"} # openssl version to build, with expected sha256 hash of .tar.gz # archive diff --git a/wheel/build_wheel.sh b/wheel/build_wheel.sh index 7e61d4a26..a94665607 100755 --- a/wheel/build_wheel.sh +++ b/wheel/build_wheel.sh @@ -137,24 +137,23 @@ export INSTALL_TEST=0 # dont install test binaries into site-packages export MACOSX_DEPLOYMENT_TARGET=10.10 export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} -SETUPTOOLS_PINNED_VERSION="" -PYYAML_PINNED_VERSION="" +SETUPTOOLS_PINNED_VERSION="=46.0.0" +PYYAML_PINNED_VERSION="=5.3" case ${desired_python} in - 3.5 | 2.7) - # setuptools and pyyaml discontinued support for python 3.5 and 2.7 - continue + 3.9) + SETUPTOOLS_PINNED_VERSION=">=46.0.0" + PYYAML_PINNED_VERSION=">=5.3" + NUMPY_PINNED_VERSION=">=1.19" ;; - 3*) - SETUPTOOLS_PINNED_VERSION="=46.0.0" - PYYAML_PINNED_VERSION="=5.3" + 3.8) + NUMPY_PINNED_VERSION="=1.17" + ;; + *) + NUMPY_PINNED_VERSION="=1.11.3" ;; esac -if [[ "$desired_python" == 3.8 ]]; then - retry conda install -yq cmake numpy=1.17 nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" ninja -else - retry conda install -yq cmake numpy==1.11.3 nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" cffi typing_extensions ninja requests -fi +retry conda install -yq cmake "numpy${NUMPY_PINNED_VERSION}" nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" cffi typing_extensions ninja requests retry conda install -yq mkl-include==2020.1 mkl-static==2020.1 -c intel retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true diff --git a/windows/build_pytorch.bat b/windows/build_pytorch.bat index 64a26e2dd..0a3e60d4e 100644 --- a/windows/build_pytorch.bat +++ b/windows/build_pytorch.bat @@ -59,6 +59,7 @@ FOR %%v IN (%DESIRED_PYTHON%) DO ( if "%%v" == "3.6" conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl>=2019" cffi pyyaml boto3 cmake ninja typing_extensions dataclasses python=%%v if "%%v" == "3.7" conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl>=2019" cffi pyyaml boto3 cmake ninja typing_extensions python=%%v if "%%v" == "3.8" conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl>=2019" pyyaml boto3 cmake ninja typing_extensions python=%%v + if "%%v" == "3.9" conda create -n py!PYTHON_VERSION_STR! -y -q -c conda-forge "numpy>=1.11" "mkl>=2019" pyyaml boto3 cmake ninja typing_extensions python=%%v if "%%v" == "3" conda create -n py!PYTHON_VERSION_STR! -y -q numpy=1.11 "mkl>=2019" pyyaml boto3 cmake ninja typing_extensions python=%%v ) endlocal From 9dc7b76114b85c96b17831aa395e36a72eba6558 Mon Sep 17 00:00:00 2001 From: Eli Uriegas <1700823+seemethere@users.noreply.github.com> Date: Thu, 5 Nov 2020 10:14:37 -0800 Subject: [PATCH 2/7] Remove older versions Co-authored-by: Nikita Shulga --- manywheel/build_scripts/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manywheel/build_scripts/build.sh b/manywheel/build_scripts/build.sh index dd5b28d22..7b374bd88 100644 --- a/manywheel/build_scripts/build.sh +++ b/manywheel/build_scripts/build.sh @@ -5,7 +5,7 @@ set -ex # Python versions to be installed in /opt/$VERSION_NO -CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"2.7.15 3.5.5 3.6.6 3.7.5 3.8.1 3.9.0"} +CPYTHON_VERSIONS=${CPYTHON_VERSIONS:-"3.6.6 3.7.5 3.8.1 3.9.0"} # openssl version to build, with expected sha256 hash of .tar.gz # archive From 67da1f824a69b06941493a4c167f7ccd1eb882b7 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 5 Nov 2020 11:09:30 -0800 Subject: [PATCH 3/7] smoke_test: Fix smoke test for conda Signed-off-by: Eli Uriegas --- smoke_test.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/smoke_test.sh b/smoke_test.sh index 4431ebfb2..930575bde 100755 --- a/smoke_test.sh +++ b/smoke_test.sh @@ -94,11 +94,18 @@ fi if [[ "$PACKAGE_TYPE" == 'conda' || "$(uname)" == 'Darwin' ]]; then # Create a new conda env in conda, or on MacOS conda create -yn test python="$py_dot" && source activate test - if [[ "$(python --version 2>&1)" == *3.6.* ]]; then - retry conda install -yq future numpy protobuf six requests dataclasses - else - retry conda install -yq future numpy protobuf six requests - fi + python_version=$(python --version 2>&1) + dependencies="numpy protobuf six requests" + case ${python_version} in + *3.6.*) + dependencies="${dependencies} future dataclasses" + ;; + # TODO: Remove this once Python 3.9 is workable through the default conda channels + *3.9.*) + dependencies="-c=conda-forge ${dependencies}" + ;; + esac + conda install -yq ${dependencies} else export PATH=/opt/python/${py_long}/bin:$PATH if [[ "$(python --version 2>&1)" == *3.6.* ]]; then From e5da1959ec99b30530c7428981f4c03ba18f5e77 Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 5 Nov 2020 12:11:09 -0800 Subject: [PATCH 4/7] wheel: Add conda-forge for python=3.9 Simplifies conda install as well to make it 1 call instead of 3 Signed-off-by: Eli Uriegas --- wheel/build_wheel.sh | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/wheel/build_wheel.sh b/wheel/build_wheel.sh index a94665607..f37b50a83 100755 --- a/wheel/build_wheel.sh +++ b/wheel/build_wheel.sh @@ -137,10 +137,12 @@ export INSTALL_TEST=0 # dont install test binaries into site-packages export MACOSX_DEPLOYMENT_TARGET=10.10 export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} +CONDA_INSTALL_EXTRA_FLAGS="" SETUPTOOLS_PINNED_VERSION="=46.0.0" PYYAML_PINNED_VERSION="=5.3" case ${desired_python} in 3.9) + CONDA_INSTALL_EXTRA_FLAGS="-c=conda-forge" SETUPTOOLS_PINNED_VERSION=">=46.0.0" PYYAML_PINNED_VERSION=">=5.3" NUMPY_PINNED_VERSION=">=1.19" @@ -153,13 +155,24 @@ case ${desired_python} in ;; esac -retry conda install -yq cmake "numpy${NUMPY_PINNED_VERSION}" nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" cffi typing_extensions ninja requests -retry conda install -yq mkl-include==2020.1 mkl-static==2020.1 -c intel -retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true - # For USE_DISTRIBUTED=1 on macOS, need libuv and pkg-config to find libuv. export USE_DISTRIBUTED=1 -retry conda install -yq libuv pkg-config + +retry conda install ${CONDA_INSTALL_EXTRA_FLAGS} -c intel -yq \ + "numpy${NUMPY_PINNED_VERSION}" \ + "pyyaml${PYYAML_PINNED_VERSION}" \ + "setuptools${SETUPTOOLS_PINNED_VERSION}" \ + cffi \ + cmake \ + ninja \ + mkl-include==2020.1 \ + mkl-static==2020.1 \ + libuv \ + pkg-config \ + nomkl \ + requests \ + typing_extensions +retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true pushd "$pytorch_rootdir" echo "Calling setup.py bdist_wheel at $(date)" From a99c05fd55fdd8dc18d36adf24a7789bc4d29e8c Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Thu, 5 Nov 2020 16:58:31 -0800 Subject: [PATCH 5/7] Add conda-forge to run_tests for Python 3.9 Signed-off-by: Eli Uriegas --- run_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index a72e43d54..678025248 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -75,7 +75,9 @@ if [[ "$package_type" == conda || "$(uname)" == Darwin ]]; then retry conda install -yq cpuonly -c pytorch fi fi - if [[ "$(python --version 2>&1)" == *3.8.* ]]; then + if [[ "$(python --version 2>&1)" == *3.9.* ]]; then + retry conda install -yq -c conda-forge future hypothesis mkl>=2018 ninja numpy>=1.15 protobuf pytest setuptools six typing_extensions pyyaml + elif [[ "$(python --version 2>&1)" == *3.8.* ]]; then retry conda install -yq future hypothesis mkl>=2018 ninja numpy>=1.15 protobuf pytest setuptools six typing_extensions pyyaml elif [[ "$(python --version 2>&1)" == *3.6.* ]]; then retry conda install -yq cffi future hypothesis mkl>=2018 ninja numpy>=1.11 protobuf pytest setuptools six typing_extensions pyyaml requests dataclasses From 42640c6de889420b6f253f87203ce1826655644c Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Fri, 6 Nov 2020 10:15:51 -0800 Subject: [PATCH 6/7] Revert "wheel: Add conda-forge for python=3.9" This reverts commit e5da1959ec99b30530c7428981f4c03ba18f5e77. --- wheel/build_wheel.sh | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/wheel/build_wheel.sh b/wheel/build_wheel.sh index f37b50a83..a94665607 100755 --- a/wheel/build_wheel.sh +++ b/wheel/build_wheel.sh @@ -137,12 +137,10 @@ export INSTALL_TEST=0 # dont install test binaries into site-packages export MACOSX_DEPLOYMENT_TARGET=10.10 export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} -CONDA_INSTALL_EXTRA_FLAGS="" SETUPTOOLS_PINNED_VERSION="=46.0.0" PYYAML_PINNED_VERSION="=5.3" case ${desired_python} in 3.9) - CONDA_INSTALL_EXTRA_FLAGS="-c=conda-forge" SETUPTOOLS_PINNED_VERSION=">=46.0.0" PYYAML_PINNED_VERSION=">=5.3" NUMPY_PINNED_VERSION=">=1.19" @@ -155,24 +153,13 @@ case ${desired_python} in ;; esac +retry conda install -yq cmake "numpy${NUMPY_PINNED_VERSION}" nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" cffi typing_extensions ninja requests +retry conda install -yq mkl-include==2020.1 mkl-static==2020.1 -c intel +retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true + # For USE_DISTRIBUTED=1 on macOS, need libuv and pkg-config to find libuv. export USE_DISTRIBUTED=1 - -retry conda install ${CONDA_INSTALL_EXTRA_FLAGS} -c intel -yq \ - "numpy${NUMPY_PINNED_VERSION}" \ - "pyyaml${PYYAML_PINNED_VERSION}" \ - "setuptools${SETUPTOOLS_PINNED_VERSION}" \ - cffi \ - cmake \ - ninja \ - mkl-include==2020.1 \ - mkl-static==2020.1 \ - libuv \ - pkg-config \ - nomkl \ - requests \ - typing_extensions -retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true +retry conda install -yq libuv pkg-config pushd "$pytorch_rootdir" echo "Calling setup.py bdist_wheel at $(date)" From acbc28ae469371137010a549e7e29edcd551fbcd Mon Sep 17 00:00:00 2001 From: Eli Uriegas Date: Fri, 6 Nov 2020 10:18:03 -0800 Subject: [PATCH 7/7] wheel: Add extra flags for conda-forge Signed-off-by: Eli Uriegas --- wheel/build_wheel.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/wheel/build_wheel.sh b/wheel/build_wheel.sh index a94665607..51ccd929c 100755 --- a/wheel/build_wheel.sh +++ b/wheel/build_wheel.sh @@ -109,10 +109,6 @@ fi wheel_filename_new="${TORCH_PACKAGE_NAME}-${build_version}${build_number_prefix}-cp${python_nodot}-none-${mac_version}.whl" ########################################################### -# Install into a fresh env -tmp_env_name="wheel_py$python_nodot" -conda create -yn "$tmp_env_name" python="$desired_python" -source activate "$tmp_env_name" # Have a separate Pytorch repo clone if [[ ! -d "$pytorch_rootdir" ]]; then @@ -139,8 +135,10 @@ export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} SETUPTOOLS_PINNED_VERSION="=46.0.0" PYYAML_PINNED_VERSION="=5.3" +EXTRA_CONDA_INSTALL_FLAGS="" case ${desired_python} in 3.9) + EXTRA_CONDA_INSTALL_FLAGS="-c=conda-forge" SETUPTOOLS_PINNED_VERSION=">=46.0.0" PYYAML_PINNED_VERSION=">=5.3" NUMPY_PINNED_VERSION=">=1.19" @@ -153,13 +151,18 @@ case ${desired_python} in ;; esac -retry conda install -yq cmake "numpy${NUMPY_PINNED_VERSION}" nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" cffi typing_extensions ninja requests -retry conda install -yq mkl-include==2020.1 mkl-static==2020.1 -c intel +# Install into a fresh env +tmp_env_name="wheel_py$python_nodot" +conda create ${EXTRA_CONDA_INSTALL_FLAGS} -yn "$tmp_env_name" python="$desired_python" +source activate "$tmp_env_name" + +retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq cmake "numpy${NUMPY_PINNED_VERSION}" nomkl "setuptools${SETUPTOOLS_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" cffi typing_extensions ninja requests +retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq mkl-include==2020.1 mkl-static==2020.1 -c intel retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true # For USE_DISTRIBUTED=1 on macOS, need libuv and pkg-config to find libuv. export USE_DISTRIBUTED=1 -retry conda install -yq libuv pkg-config +retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq libuv pkg-config pushd "$pytorch_rootdir" echo "Calling setup.py bdist_wheel at $(date)"