diff --git a/packaging/conda/build_vision.sh b/packaging/conda/build_vision.sh index 3061e4740bb..8122648a722 100755 --- a/packaging/conda/build_vision.sh +++ b/packaging/conda/build_vision.sh @@ -10,15 +10,56 @@ retry () { $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) } -if [ "$#" -ne 1 ]; then - echo "Illegal number of parameters. Pass cuda version" - echo "CUDA version should be M.m with no dot, e.g. '8.0' or 'cpu'" - exit 1 +# Parse arguments and determmine version +########################################################### +if [[ -n "$DESIRED_CUDA" && -n "$TORCHVISION_BUILD_VERSION" && -n "$TORCHVISION_BUILD_NUMBER" ]]; then + desired_cuda="$DESIRED_CUDA" + build_version="$PYTORCH_BUILD_VERSION" + build_number="$PYTORCH_BUILD_NUMBER" +else + if [ "$#" -ne 3 ]; then + echo "Illegal number of parameters. Pass cuda version, pytorch version, build number" + echo "CUDA version should be Mm with no dot, e.g. '80'" + echo "DESIRED_PYTHON should be M.m, e.g. '2.7'" + exit 1 + fi + + desired_cuda="$1" + build_version="$2" + build_number="$3" fi -desired_cuda="$1" +if [[ "$desired_cuda" != cpu ]]; then + desired_cuda="$(echo $desired_cuda | tr -d cuda. )" +fi +echo "Building cuda version $desired_cuda and torchvision version: $build_version build_number: $build_number" -export TORCHVISION_BUILD_VERSION="0.3.0" -export TORCHVISION_BUILD_NUMBER=1 +if [[ "$desired_cuda" == 'cpu' ]]; then + cpu_only=1 + cuver="cpu" +else + # Switch desired_cuda to be M.m to be consistent with other scripts in + # pytorch/builder + export FORCE_CUDA=1 + cuda_nodot="$desired_cuda" + + if [[ ${#cuda_nodot} -eq 2 ]]; then + desired_cuda="${desired_cuda:0:1}.${desired_cuda:1:1}" + elif [[ ${#cuda_nodot} -eq 3 ]]; then + desired_cuda="${desired_cuda:0:2}.${desired_cuda:2:1}" + else + echo "unknown cuda version $cuda_nodot" + exit 1 + fi + + cuver="cu$cuda_nodot" +fi + +export TORCHVISION_BUILD_VERSION=$build_version +export TORCHVISION_BUILD_NUMBER=$build_number + +if [[ -z "$DESIRED_PYTHON" ]]; then + DESIRED_PYTHON=('3.5' '3.6' '3.7') +fi SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" @@ -26,69 +67,147 @@ if [[ -z "$WIN_PACKAGE_WORK_DIR" ]]; then WIN_PACKAGE_WORK_DIR="$(echo $(pwd -W) | tr '/' '\\')\\tmp_conda_$(date +%H%M%S)" fi -if [[ "$OSTYPE" == "msys" ]]; then - mkdir -p "$WIN_PACKAGE_WORK_DIR" || true - vision_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/torchvision-src" - git config --system core.longpaths true -else - vision_rootdir="$(pwd)/torchvision-src" -fi +mkdir -p "$WIN_PACKAGE_WORK_DIR" || true +vision_rootdir="$(realpath ${WIN_PACKAGE_WORK_DIR})/torchvision-src" +git config --system core.longpaths true if [[ ! -d "$vision_rootdir" ]]; then rm -rf "$vision_rootdir" git clone "https://github.com/pytorch/vision" "$vision_rootdir" pushd "$vision_rootdir" - git checkout v$TORCHVISION_BUILD_VERSION + git checkout $PYTORCH_BRANCH popd fi cd "$SOURCE_DIR" -if [[ "$OSTYPE" == "msys" ]]; then - export tmp_conda="${WIN_PACKAGE_WORK_DIR}\\conda" - export miniconda_exe="${WIN_PACKAGE_WORK_DIR}\\miniconda.exe" - rm -rf "$tmp_conda" - rm -f "$miniconda_exe" - curl -sSk https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe" - "$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe" - pushd $tmp_conda - export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH" - popd - # We have to skip 3.17 because of the following bug. - # https://github.com/conda/conda-build/issues/3285 - retry conda install -yq conda-build -fi +export tmp_conda="${WIN_PACKAGE_WORK_DIR}\\conda" +export miniconda_exe="${WIN_PACKAGE_WORK_DIR}\\miniconda.exe" +rm -rf "$tmp_conda" +rm -f "$miniconda_exe" +curl -sSk https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "$miniconda_exe" +"$SOURCE_DIR/install_conda.bat" && rm "$miniconda_exe" +pushd $tmp_conda +export PATH="$(pwd):$(pwd)/Library/usr/bin:$(pwd)/Library/bin:$(pwd)/Scripts:$(pwd)/bin:$PATH" +popd +retry conda install -yq conda-build -ANACONDA_USER=pytorch +ANACONDA_USER=pytorch-nightly conda config --set anaconda_upload no export TORCHVISION_PACKAGE_SUFFIX="" if [[ "$desired_cuda" == 'cpu' ]]; then export CONDA_CUDATOOLKIT_CONSTRAINT="" + export CONDA_CPUONLY_FEATURE="- cpuonly # [not osx]" export CUDA_VERSION="None" - if [[ "$OSTYPE" != "darwin"* ]]; then - export TORCHVISION_PACKAGE_SUFFIX="-cpu" - fi else + export CONDA_CPUONLY_FEATURE="" . ./switch_cuda_version.sh $desired_cuda - if [[ "$desired_cuda" == "10.0" ]]; then - export CONDA_CUDATOOLKIT_CONSTRAINT=" - cudatoolkit >=10.0,<10.1 # [not osx]" + if [[ "$desired_cuda" == "10.1" ]]; then + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.1,<10.2 # [not osx]" + elif [[ "$desired_cuda" == "10.0" ]]; then + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=10.0,<10.1 # [not osx]" + elif [[ "$desired_cuda" == "9.2" ]]; then + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.2,<9.3 # [not osx]" elif [[ "$desired_cuda" == "9.0" ]]; then - export CONDA_CUDATOOLKIT_CONSTRAINT=" - cudatoolkit >=9.0,<9.1 # [not osx]" + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=9.0,<9.1 # [not osx]" + elif [[ "$desired_cuda" == "8.0" ]]; then + export CONDA_CUDATOOLKIT_CONSTRAINT="- cudatoolkit >=8.0,<8.1 # [not osx]" else - echo "unhandled desired_cuda: $desired_cuda" - exit 1 + echo "unhandled desired_cuda: $desired_cuda" + exit 1 fi fi -if [[ "$OSTYPE" == "msys" ]]; then - time conda build -c $ANACONDA_USER --no-anaconda-upload vs2017 +if [[ -z "$PYTORCH_VERSION" ]]; then + export CONDA_CHANNEL_FLAGS="-c pytorch-nightly" + export PYTORCH_VERSION="$(conda search --json 'pytorch[channel=pytorch-nightly]' | \ + python -c "import os, sys, json, re; cuver = '$cuver'; \ + cuver = cuver.replace('cu', 'cuda') if cuver != 'cpu' else cuver; \ + print(re.sub(r'\\+.*$', '', \ + [x['version'] for x in json.load(sys.stdin)['pytorch'] \ + if (x['platform'] == 'darwin' or cuver in x['fn']) \ + and 'py' + os.environ['DESIRED_PYTHON'] in x['fn']][-1]))")" + if [[ -z "$PYTORCH_VERSION" ]]; then + echo "PyTorch version auto detection failed" + echo "No package found for desired_cuda=$desired_cuda and DESIRED_PYTHON=$DESIRED_PYTHON" + exit 1 + fi +else + export CONDA_CHANNEL_FLAGS="-c pytorch -c pytorch-nightly" +fi +if [[ "$desired_cuda" == 'cpu' ]]; then + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==$PYTORCH_VERSION" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==$PYTORCH_VERSION" else - time conda build -c $ANACONDA_USER --no-anaconda-upload --python 2.7 torchvision + export CONDA_PYTORCH_BUILD_CONSTRAINT="- pytorch==${PYTORCH_VERSION}" + export CONDA_PYTORCH_CONSTRAINT="- pytorch==${PYTORCH_VERSION}" fi -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.5 torchvision -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.6 torchvision -time conda build -c $ANACONDA_USER --no-anaconda-upload --python 3.7 torchvision + +# Loop through all Python versions to build a package for each +for py_ver in "${DESIRED_PYTHON[@]}"; do + build_string="py${py_ver}_${build_string_suffix}" + folder_tag="${build_string}_$(date +'%Y%m%d')" + + # Create the conda package into this temporary folder. This is so we can find + # the package afterwards, as there's no easy way to extract the final filename + # from conda-build + output_folder="out_$folder_tag" + rm -rf "$output_folder" + mkdir "$output_folder" + + # We need to build the compiler activation scripts first on Windows + time VSDEVCMD_ARGS=${VSDEVCMD_ARGS[@]} \ + conda build -c "$ANACONDA_USER" \ + --no-anaconda-upload \ + --output-folder "$output_folder" \ + ../vs2017 + + conda config --set anaconda_upload no + echo "Calling conda-build at $(date)" + if [[ "$desired_cuda" == "9.2" ]]; then + time CMAKE_ARGS=${CMAKE_ARGS[@]} \ + BUILD_VERSION="$TORCHVISION_BUILD_VERSION" \ + CU_VERSION="$cuver" \ + SOURCE_ROOT_DIR="$vision_rootdir" \ + conda build -c "$ANACONDA_USER" \ + -c defaults \ + -c conda-forge \ + -c "numba/label/dev" \ + --no-anaconda-upload \ + --python "$py_ver" \ + --output-folder "$output_folder" \ + --no-verify \ + --no-test \ + ../torchvision + else + time CMAKE_ARGS=${CMAKE_ARGS[@]} \ + BUILD_VERSION="$TORCHVISION_BUILD_VERSION" \ + CU_VERSION="$cuver" \ + SOURCE_ROOT_DIR="$vision_rootdir" \ + conda build -c "$ANACONDA_USER" \ + -c defaults \ + -c conda-forge \ + --no-anaconda-upload \ + --python "$py_ver" \ + --output-folder "$output_folder" \ + --no-verify \ + --no-test \ + ../torchvision + fi + echo "Finished conda-build at $(date)" + + # Extract the package for testing + ls -lah "$output_folder" + built_package="$(find $output_folder/ -name '*torchvision*.tar.bz2')" + + # Copy the built package to the host machine for persistence before testing + if [[ -n "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then + mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true + cp "$built_package" "$PYTORCH_FINAL_PACKAGE_DIR/" + fi +done + set +e diff --git a/packaging/torchvision/bld.bat b/packaging/torchvision/bld.bat index 14f6935fba8..0f1265c25f6 100644 --- a/packaging/torchvision/bld.bat +++ b/packaging/torchvision/bld.bat @@ -16,6 +16,7 @@ set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v%desired_cuda% set CUDA_BIN_PATH=%CUDA_PATH%\bin set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr if "%desired_cuda%" == "9.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50 +if "%desired_cuda%" == "9.2" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50 if "%desired_cuda%" == "10.0" set NVCC_FLAGS=%NVCC_FLAGS% -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50 :cuda_flags_end diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index fdcfbf78dc8..20397c40f20 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -14,6 +14,7 @@ requirements: - setuptools {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT') }} {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} + {{ environ.get('CONDA_CPUONLY_FEATURE') }} run: - python @@ -31,7 +32,7 @@ build: - FORCE_CUDA - NVCC_FLAGS features: - {{ CONDA_CPUONLY_FEATURE }} + {{ environ.get('CONDA_CPUONLY_FEATURE') }} test: imports: diff --git a/packaging/vs2017/activate.bat b/packaging/vs2017/activate.bat new file mode 100644 index 00000000000..ccecfc25442 --- /dev/null +++ b/packaging/vs2017/activate.bat @@ -0,0 +1,44 @@ +:: Set env vars that tell distutils to use the compiler that we put on path +SET DISTUTILS_USE_SDK=1 +SET MSSdk=1 + +SET "VS_VERSION=15.0" +SET "VS_MAJOR=15" +SET "VS_YEAR=2017" + +set "MSYS2_ARG_CONV_EXCL=/AI;/AL;/OUT;/out" +set "MSYS2_ENV_CONV_EXCL=CL" + +:: For Python 3.5+, ensure that we link with the dynamic runtime. See +:: http://stevedower.id.au/blog/building-for-python-3-5-part-two/ for more info +set "PY_VCRUNTIME_REDIST=%PREFIX%\\bin\\vcruntime140.dll" + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VSINSTALLDIR=%%i\" + goto :vswhere + ) +) + +:vswhere + +:: Shorten PATH to avoid the `input line too long` error. +SET MyPath=%PATH% + +setlocal EnableDelayedExpansion + +SET TempPath="%MyPath:;=";"%" +SET var= +FOR %%a IN (%TempPath%) DO ( + IF EXIST %%~sa ( + SET "var=!var!;%%~sa" + ) +) + +set "TempPath=!var:~1!" +endlocal & set "PATH=%TempPath%" + +:: Shorten current directory too +FOR %%A IN (.) DO CD "%%~sA" + +:: other things added by install_activate.bat at package build time diff --git a/packaging/vs2017/conda_build_config.yaml b/packaging/vs2017/conda_build_config.yaml new file mode 100644 index 00000000000..5188bb0ebec --- /dev/null +++ b/packaging/vs2017/conda_build_config.yaml @@ -0,0 +1,24 @@ +blas_impl: + - mkl # [x86_64] +c_compiler: + - vs2017 # [win] +cxx_compiler: + - vs2017 # [win] +python: + - 3.5 + - 3.6 +# This differs from target_platform in that it determines what subdir the compiler +# will target, not what subdir the compiler package will be itself. +# For example, we need a win-64 vs2008_win-32 package, so that we compile win-32 +# code on win-64 miniconda. +cross_compiler_target_platform: + - win-64 # [win] +target_platform: + - win-64 # [win] +vc: + - 14 +zip_keys: + - # [win] + - vc # [win] + - c_compiler # [win] + - cxx_compiler # [win] diff --git a/packaging/vs2017/install_activate.bat b/packaging/vs2017/install_activate.bat new file mode 100644 index 00000000000..de0e6ff3c52 --- /dev/null +++ b/packaging/vs2017/install_activate.bat @@ -0,0 +1,30 @@ +set YEAR=2017 +set VER=15 + +mkdir "%PREFIX%\etc\conda\activate.d" +COPY "%RECIPE_DIR%\activate.bat" "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + +IF "%cross_compiler_target_platform%" == "win-64" ( + set "target_platform=amd64" + echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR% Win64" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + IF "%VSDEVCMD_ARGS%" == "" ( + echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + ) ELSE ( + echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + ) + echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + ) else ( + set "target_platform=x86" + echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo CALL "VC\Auxiliary\Build\vcvars32.bat" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat" + echo popd + ) + diff --git a/packaging/vs2017/install_runtime.bat b/packaging/vs2017/install_runtime.bat new file mode 100644 index 00000000000..5163c16cf24 --- /dev/null +++ b/packaging/vs2017/install_runtime.bat @@ -0,0 +1,49 @@ +set VC_PATH=x86 +if "%ARCH%"=="64" ( + set VC_PATH=x64 +) + +set MSC_VER=2017 + +rem :: This should always be present for VC installed with VS. Not sure about VC installed with Visual C++ Build Tools 2015 +rem FOR /F "usebackq tokens=3*" %%A IN (`REG QUERY "HKEY_LOCAL_MACHINE\Software\Microsoft\DevDiv\VC\Servicing\14.0\IDE.x64" /v UpdateVersion`) DO ( +rem set SP=%%A +rem ) + +rem if not "%SP%" == "%PKG_VERSION%" ( +rem echo "Version detected from registry: %SP%" +rem echo "does not match version of package being built (%PKG_VERSION%)" +rem echo "Do you have current updates for VS 2015 installed?" +rem exit 1 +rem ) + + +REM ========== REQUIRES Win 10 SDK be installed, or files otherwise copied to location below! +robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%" "%LIBRARY_BIN%" *.dll /E +robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%" "%PREFIX%" *.dll /E +if %ERRORLEVEL% GEQ 8 exit 1 + +REM ========== This one comes from visual studio 2017 +set "VC_VER=141" + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + goto :eof + ) +) + +@setlocal +call "%VS15VARSALL%" x64 + +set "REDIST_ROOT=%VCToolsRedistDir%%VC_PATH%" + +robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%LIBRARY_BIN%" *.dll /E +if %ERRORLEVEL% LSS 8 exit 0 +robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%PREFIX%" *.dll /E +if %ERRORLEVEL% LSS 8 exit 0 +robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%LIBRARY_BIN%" *.dll /E +if %ERRORLEVEL% LSS 8 exit 0 +robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%PREFIX%" *.dll /E +if %ERRORLEVEL% LSS 8 exit 0 +@endlocal diff --git a/packaging/vs2017/meta.yaml b/packaging/vs2017/meta.yaml new file mode 100644 index 00000000000..34f4860ba85 --- /dev/null +++ b/packaging/vs2017/meta.yaml @@ -0,0 +1,45 @@ +{% set vcver="14.1" %} +{% set vcfeature="14" %} +{% set vsyear="2017" %} +{% set fullver="15.4.27004.2010" %} + +package: + name: vs{{ vsyear }} + version: {{ fullver }} + +build: + skip: True [not win] + script_env: + - VSDEVCMD_ARGS # [win] + +outputs: + - name: vs{{ vsyear }}_{{ cross_compiler_target_platform }} + script: install_activate.bat + track_features: + # VS 2017 is binary-compatible with VS 2015/vc14. Tools are "v141". + strong: + - vc{{ vcfeature }} + run_exports: + - vc {{ vcver }} + about: + summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler + license: BSD 3-clause + - name: vs{{ vsyear }}_runtime + script: install_runtime.bat + - name: vc + version: {{ vcver }} + track_features: + - vc{{ vcfeature }} + requirements: + run: + - {{ pin_subpackage('vs' ~ vsyear ~ '_runtime') }} + about: + home: https://github.com/conda/conda/wiki/VC-features + license: Modified BSD License (3-clause) + license_family: BSD + summary: A meta-package to track VC features. + description: | + This metapackage is used to activate vc features without + depending on Python. + doc_url: https://github.com/conda/conda/wiki/VC-features + dev_url: https://github.com/conda/conda/wiki/VC-features diff --git a/packaging/windows/azure-pipelines-ci.yml b/packaging/windows/azure-pipelines-ci.yml new file mode 100644 index 00000000000..6f9f3468cfe --- /dev/null +++ b/packaging/windows/azure-pipelines-ci.yml @@ -0,0 +1,11 @@ + +# Turn off auto builds for commits +trigger: none +pr: none + +jobs: +- template: templates/build_task.yml + parameters: + package: 'Wheels' + spec: 'CPU' + msagent: true diff --git a/packaging/windows/azure-pipelines.yml b/packaging/windows/azure-pipelines.yml new file mode 100644 index 00000000000..d0240570012 --- /dev/null +++ b/packaging/windows/azure-pipelines.yml @@ -0,0 +1,35 @@ + +# Turn off auto builds for commits +trigger: none +pr: none + +jobs: +- template: templates/auth_task.yml + +- template: templates/build_task.yml + parameters: + package: 'Wheels' + spec: 'CPU' + msagent: true + +- template: templates/build_task.yml + parameters: + package: 'Conda' + spec: 'CPU' + msagent: true + +- template: templates/build_task.yml + parameters: + package: 'Wheels' + spec: 'CUDA' + msagent: true + +- template: templates/build_task.yml + parameters: + package: 'Conda' + spec: 'CUDA' + msagent: true + +- template: templates/linux_build_task.yml + parameters: + msagent: $(ms.hosted.agent.cpu) diff --git a/packaging/windows/build_vision.bat b/packaging/windows/build_vision.bat index c7b65dc04f1..162dba44917 100644 --- a/packaging/windows/build_vision.bat +++ b/packaging/windows/build_vision.bat @@ -23,18 +23,21 @@ set CUDA_VERSION=%~1 set TORCHVISION_BUILD_VERSION=%~2 set TORCHVISION_BUILD_NUMBER=%~3 +set BUILD_VERSION=%TORCHVISION_BUILD_VERSION% + :env_end if NOT "%CUDA_VERSION%" == "cpu" ( set CUDA_PREFIX=cuda%CUDA_VERSION% set CUVER=cu%CUDA_VERSION% + set FORCE_CUDA=1 ) else ( set CUDA_PREFIX=cpu set CUVER=cpu ) set BUILD_VISION=1 -set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index +REM set TORCH_WHEEL=torch -f https://download.pytorch.org/whl/%CUVER%/stable.html --no-index IF "%DESIRED_PYTHON%" == "" set DESIRED_PYTHON=3.5;3.6;3.7 set DESIRED_PYTHON_PREFIX=%DESIRED_PYTHON:.=% @@ -61,8 +64,35 @@ FOR %%v IN (%DESIRED_PYTHON%) DO ( set PYTHON_VERSION_STR=%%v set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=! conda remove -n py!PYTHON_VERSION_STR! --all -y || rmdir %CONDA_HOME%\envs\py!PYTHON_VERSION_STR! /s - conda create -n py!PYTHON_VERSION_STR! -y -q numpy>=1.11 mkl>=2018 python=%%v + conda create -n py!PYTHON_VERSION_STR! -y -q -c defaults -c conda-forge numpy>=1.11 mkl>=2018 python=%%v ca-certificates scipy av +) + +:: Uncomment for stable releases +:: FOR %%v IN (%DESIRED_PYTHON%) DO ( +:: set PYTHON_VERSION_STR=%%v +:: set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=! +:: set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%" + +:: if "%CUDA_VERSION%" == "100" ( +:: set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl +:: ) else ( +:: set TORCH_WHEEL=https://download.pytorch.org/whl/%CUVER%/torch-1.2.0%%2B%CUVER%-cp!PYTHON_VERSION_STR!-cp!PYTHON_VERSION_STR!m-win_amd64.whl +:: ) +:: echo Installing !TORCH_WHEEL!... +:: pip install "!TORCH_WHEEL!" +:: ) + +:: Uncomment for nightly releases +FOR %%v IN (%DESIRED_PYTHON%) DO ( + set PYTHON_VERSION_STR=%%v + set PYTHON_VERSION_STR=!PYTHON_VERSION_STR:.=! + set "PATH=%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\scripts;%CONDA_HOME%\envs\py!PYTHON_VERSION_STR!\Library\bin;%ORIG_PATH%" + + set TORCH_WHEEL=torch --pre -f https://download.pytorch.org/whl/nightly/%CUVER%/torch_nightly.html + echo Installing !TORCH_WHEEL!... + pip install !TORCH_WHEEL! ) + endlocal if "%DEBUG%" == "1" ( @@ -71,11 +101,22 @@ if "%DEBUG%" == "1" ( set BUILD_TYPE=release ) +:: Install sccache +if "%USE_SCCACHE%" == "1" ( + mkdir %CD%\tmp_bin + curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output %CD%\tmp_bin\sccache.exe + if not "%CUDA_VERSION%" == "" ( + copy %CD%\tmp_bin\sccache.exe %CD%\tmp_bin\nvcc.exe + + set CUDA_NVCC_EXECUTABLE=%CD%\tmp_bin\nvcc + set "PATH=%CD%\tmp_bin;%PATH%" + ) +) + for %%v in (%DESIRED_PYTHON_PREFIX%) do ( :: Activate Python Environment set PYTHON_PREFIX=%%v set "PATH=%CONDA_HOME%\envs\%%v;%CONDA_HOME%\envs\%%v\scripts;%CONDA_HOME%\envs\%%v\Library\bin;%ORIG_PATH%" - pip install %TORCH_WHEEL% @setlocal :: Set Flags if NOT "%CUDA_VERSION%"=="cpu" ( diff --git a/packaging/windows/cuda92.bat b/packaging/windows/cuda92.bat new file mode 100644 index 00000000000..0bfcdc8e463 --- /dev/null +++ b/packaging/windows/cuda92.bat @@ -0,0 +1,59 @@ +@echo off + +IF NOT "%BUILD_VISION%" == "" ( + set MODULE_NAME=vision +) ELSE ( + set MODULE_NAME=pytorch +) + +IF NOT EXIST "setup.py" IF NOT EXIST "%MODULE_NAME%" ( + call internal\clone.bat + cd .. + IF ERRORLEVEL 1 goto eof +) ELSE ( + call internal\clean.bat +) + +call internal\check_deps.bat +IF ERRORLEVEL 1 goto eof + +REM Check for optional components + +set USE_CUDA= +set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 + +IF "%NVTOOLSEXT_PATH%"=="" ( + echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing + exit /b 1 + goto optcheck +) + +IF "%CUDA_PATH_V9_2%"=="" ( + echo CUDA 9.2 not found, failing + exit /b 1 +) ELSE ( + IF "%BUILD_VISION%" == "" ( + set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0 + set TORCH_NVCC_FLAGS=-Xfatbin -compress-all + ) ELSE ( + set NVCC_FLAGS=-D__CUDA_NO_HALF_OPERATORS__ --expt-relaxed-constexpr -gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_50,code=compute_50 + ) + + set "CUDA_PATH=%CUDA_PATH_V9_2%" + set "PATH=%CUDA_PATH_V9_2%\bin;%PATH%" +) + +:optcheck + +IF "%BUILD_VISION%" == "" ( + call internal\check_opts.bat + IF ERRORLEVEL 1 goto eof + + call internal\copy.bat + IF ERRORLEVEL 1 goto eof +) + +call internal\setup.bat +IF ERRORLEVEL 1 goto eof + +:eof diff --git a/packaging/windows/internal/7z_install.bat b/packaging/windows/internal/7z_install.bat deleted file mode 100644 index d5a1156360d..00000000000 --- a/packaging/windows/internal/7z_install.bat +++ /dev/null @@ -1,9 +0,0 @@ -@echo off - -curl -k https://www.7-zip.org/a/7z1805-x64.exe -O -if errorlevel 1 exit /b 1 - -start /wait 7z1805-x64.exe /S -if errorlevel 1 exit /b 1 - -set "PATH=%ProgramFiles%\7-Zip;%PATH%" diff --git a/packaging/windows/internal/clone.bat b/packaging/windows/internal/clone.bat index f93271978c6..4ba181fa804 100644 --- a/packaging/windows/internal/clone.bat +++ b/packaging/windows/internal/clone.bat @@ -44,15 +44,11 @@ set PYTORCH_BRANCH=%last_commit% :latest_end -IF "%BUILD_VISION%" == "" ( - IF "%PYTORCH_BRANCH%" == "" ( - set PYTORCH_BRANCH=v%PYTORCH_BUILD_VERSION% - ) - git checkout %PYTORCH_BRANCH% - IF ERRORLEVEL 1 git checkout tags/%PYTORCH_BRANCH% -) ELSE ( - git checkout v%TORCHVISION_BUILD_VERSION% +IF "%PYTORCH_BRANCH%" == "" ( + set PYTORCH_BRANCH=v%TORCHVISION_BUILD_VERSION% ) +git checkout %PYTORCH_BRANCH% +IF ERRORLEVEL 1 git checkout tags/%PYTORCH_BRANCH% :submodule diff --git a/packaging/windows/internal/cuda_install.bat b/packaging/windows/internal/cuda_install.bat index b6ea9e23739..a85018b11d7 100644 --- a/packaging/windows/internal/cuda_install.bat +++ b/packaging/windows/internal/cuda_install.bat @@ -1,4 +1,9 @@ -@echo off +@echo on + +if "%CUDA_VERSION%" == "cpu" ( + echo Skipping for CPU builds + exit /b 0 +) set SRC_DIR=%~dp0\.. @@ -9,87 +14,60 @@ set CUDA_VER_MAJOR=%CUDA_VERSION:~0,-1% set CUDA_VER_MINOR=%CUDA_VERSION:~-1,1% set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR% -IF %CUDA_VER% LEQ 90 ( - set "NVCC_PACKAGE=compiler_%CUDA_VERSION_STR%" -) ELSE ( - set "NVCC_PACKAGE=nvcc_%CUDA_VERSION_STR%" -) - -IF %CUDA_VER% EQU 80 goto cuda80 -IF %CUDA_VER% EQU 90 goto cuda90 -IF %CUDA_VER% EQU 91 goto cuda91 -IF %CUDA_VER% EQU 92 goto cuda92 -IF %CUDA_VER% EQU 100 goto cuda100 +if %CUDA_VER% EQU 92 goto cuda92 +if %CUDA_VER% EQU 100 goto cuda100 echo CUDA %CUDA_VERSION_STR% is not supported exit /b 1 -:cuda80 - -echo CUDA 8.0 is not supported -exit /b 1 - -:cuda90 -IF NOT EXIST "%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z" ( - curl -k -L https://www.dropbox.com/s/z5b7ryz0zrimntl/cuda_9.0.176_windows.7z?dl=1 --output "%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z" +:cuda92 +if not exist "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe" ( + curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_9.2.148_win10.exe --output "%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe" if errorlevel 1 exit /b 1 - set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.0.176_windows.7z" - set "NVCC_PACKAGE=compiler_%CUDA_VERSION_STR%" + set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.2.148_win10.exe" + set "ARGS=nvcc_9.2 cuobjdump_9.2 nvprune_9.2 cupti_9.2 cublas_9.2 cublas_dev_9.2 cudart_9.2 cufft_9.2 cufft_dev_9.2 curand_9.2 curand_dev_9.2 cusolver_9.2 cusolver_dev_9.2 cusparse_9.2 cusparse_dev_9.2 nvgraph_9.2 nvgraph_dev_9.2 npp_9.2 npp_dev_9.2 nvrtc_9.2 nvrtc_dev_9.2 nvml_dev_9.2" ) -IF NOT EXIST "%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip" ( - curl -k -L https://www.dropbox.com/s/6p0xyqh472nu8m1/cudnn-9.0-windows7-x64-v7.zip?dl=1 --output "%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip" +if not exist "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip" ( + curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-9.2-windows10-x64-v7.2.1.38.zip --output "%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip" if errorlevel 1 exit /b 1 - set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.0-windows7-x64-v7.zip" + set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.2-windows10-x64-v7.2.1.38.zip" ) goto cuda_common -:cuda91 +:cuda100 -IF NOT EXIST "%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z" ( - curl -k -L https://www.dropbox.com/s/7a4sbq0dln6v7t2/cuda_9.1.85_windows.7z?dl=1 --output "%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z" +if not exist "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe" ( + curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cuda_10.0.130_411.31_win10.exe --output "%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe" if errorlevel 1 exit /b 1 - set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_9.1.85_windows.7z" - set "NVCC_PACKAGE=nvcc_%CUDA_VERSION_STR%" + set "CUDA_SETUP_FILE=%SRC_DIR%\temp_build\cuda_10.0.130_411.31_win10.exe" + set "ARGS=nvcc_10.0 cuobjdump_10.0 nvprune_10.0 cupti_10.0 cublas_10.0 cublas_dev_10.0 cudart_10.0 cufft_10.0 cufft_dev_10.0 curand_10.0 curand_dev_10.0 cusolver_10.0 cusolver_dev_10.0 cusparse_10.0 cusparse_dev_10.0 nvgraph_10.0 nvgraph_dev_10.0 npp_10.0 npp_dev_10.0 nvrtc_10.0 nvrtc_dev_10.0 nvml_dev_10.0" ) -IF NOT EXIST "%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip" ( - curl -k -L https://www.dropbox.com/s/e0prhgsrbyfi4ov/cudnn-9.1-windows7-x64-v7.zip?dl=1 --output "%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip" +if not exist "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip" ( + curl -k -L https://ossci-windows.s3.amazonaws.com/win2016/cudnn-10.0-windows10-x64-v7.4.1.5.zip --output "%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip" if errorlevel 1 exit /b 1 - set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-9.1-windows7-x64-v7.zip" + set "CUDNN_SETUP_FILE=%SRC_DIR%\temp_build\cudnn-10.0-windows10-x64-v7.4.1.5.zip" ) goto cuda_common -:cuda92 - -echo CUDA 9.2 is not supported -exit /b 1 - -:cuda100 - -echo CUDA 10.0 is not supported -exit /b 1 - :cuda_common -set "CUDA_PREFIX=cuda%CUDA_VERSION%" - -IF NOT EXIST "%SRC_DIR%\temp_build\NvToolsExt.7z" ( +if not exist "%SRC_DIR%\temp_build\NvToolsExt.7z" ( curl -k -L https://www.dropbox.com/s/9mcolalfdj4n979/NvToolsExt.7z?dl=1 --output "%SRC_DIR%\temp_build\NvToolsExt.7z" if errorlevel 1 exit /b 1 ) echo Installing CUDA toolkit... - 7z x %CUDA_SETUP_FILE% -o"%SRC_DIR%\temp_build\cuda" pushd "%SRC_DIR%\temp_build\cuda" -dir -start /wait setup.exe -s %NVCC_PACKAGE% cublas_%CUDA_VERSION_STR% cublas_dev_%CUDA_VERSION_STR% cudart_%CUDA_VERSION_STR% curand_%CUDA_VERSION_STR% curand_dev_%CUDA_VERSION_STR% cusparse_%CUDA_VERSION_STR% cusparse_dev_%CUDA_VERSION_STR% nvrtc_%CUDA_VERSION_STR% nvrtc_dev_%CUDA_VERSION_STR% cufft_%CUDA_VERSION_STR% cufft_dev_%CUDA_VERSION_STR% +start /wait setup.exe -s %ARGS% popd + echo Installing VS integration... -xcopy /Y "%SRC_DIR%\temp_build\cuda\_vs\*.*" "c:\Program Files (x86)\MSBuild\Microsoft.Cpp\v4.0\V140\BuildCustomizations" +xcopy /Y "%SRC_DIR%\temp_build\cuda\CUDAVisualStudioIntegration\extras\visual_studio_integration\MSBuildExtensions\*.*" "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\Common7\IDE\VC\VCTargets\BuildCustomizations" echo Installing NvToolsExt... 7z x %SRC_DIR%\temp_build\NvToolsExt.7z -o"%SRC_DIR%\temp_build\NvToolsExt" @@ -100,23 +78,22 @@ xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\bin\x64\*.*" "%ProgramFiles%\NVIDIA Co xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\include\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\include" xcopy /Y "%SRC_DIR%\temp_build\NvToolsExt\lib\x64\*.*" "%ProgramFiles%\NVIDIA Corporation\NvToolsExt\lib\x64" +echo Setting up environment... +set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\libnvvp;%PATH%" +set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%" +set "CUDA_PATH_V%CUDA_VER_MAJOR%_%CUDA_VER_MINOR%=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%" +set "NVTOOLSEXT_PATH=%ProgramFiles%\NVIDIA Corporation\NvToolsExt\bin\x64" + +if not exist "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin\nvcc.exe" ( + echo CUDA %CUDA_VERSION_STR% installed failed. + exit /b 1 +) + echo Installing cuDNN... 7z x %CUDNN_SETUP_FILE% -o"%SRC_DIR%\temp_build\cudnn" xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\bin\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin" xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\lib\x64\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\lib\x64" xcopy /Y "%SRC_DIR%\temp_build\cudnn\cuda\include\*.*" "%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\include" -echo Setting up environment... -set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%\libnvvp;%PATH%" -set "CUDA_PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%" -set "CUDA_PATH_V%CUDA_VER_MAJOR%_%CUDA_VER_MINOR%=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v%CUDA_VERSION_STR%" -set "NVTOOLSEXT_PATH=%ProgramFiles%\NVIDIA Corporation\NvToolsExt\" - echo Cleaning temp files -rd /s /q "%SRC_DIR%\temp_build" -pushd %TEMP% -rd /s /q . -popd - -echo Using VS2015 as NVCC compiler -set "CUDAHOSTCXX=%VS140COMNTOOLS%\..\..\VC\bin\amd64\cl.exe" +rd /s /q "%SRC_DIR%\temp_build" || ver > nul diff --git a/packaging/windows/internal/dep_install.bat b/packaging/windows/internal/dep_install.bat new file mode 100644 index 00000000000..db665a99f26 --- /dev/null +++ b/packaging/windows/internal/dep_install.bat @@ -0,0 +1,14 @@ +@echo off + +REM curl -k https://www.7-zip.org/a/7z1805-x64.exe -O +REM if errorlevel 1 exit /b 1 + +REM start /wait 7z1805-x64.exe /S +REM if errorlevel 1 exit /b 1 + +REM set "PATH=%ProgramFiles%\7-Zip;%PATH%" + +choco feature disable --name showDownloadProgress +choco feature enable --name allowGlobalConfirmation + +choco install curl 7zip diff --git a/packaging/windows/internal/env_fix.bat b/packaging/windows/internal/env_fix.bat new file mode 100644 index 00000000000..dd0aaf5f2d5 --- /dev/null +++ b/packaging/windows/internal/env_fix.bat @@ -0,0 +1,31 @@ +@echo off + +:: Caution: Please don't use this script locally +:: It may destroy your build environment. + +setlocal + +IF NOT EXIST "%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" ( + echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows + exit /b 1 +) + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,16^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VS15INSTALLDIR=%%i" + set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + goto vswhere + ) +) + +:vswhere + +IF "%VS15VCVARSALL%"=="" ( + echo Visual Studio 2017 C++ BuildTools is required to compile PyTorch on Windows + exit /b 1 +) + +call "%VS15VCVARSALL%" x86_amd64 +for /f "usebackq tokens=*" %%i in (`where link.exe`) do move "%%i" "%%i.bak" + +endlocal diff --git a/packaging/windows/internal/nightly_defaults.bat b/packaging/windows/internal/nightly_defaults.bat new file mode 100644 index 00000000000..208d8b85e78 --- /dev/null +++ b/packaging/windows/internal/nightly_defaults.bat @@ -0,0 +1,200 @@ +@echo on + +if "%~1"=="" goto arg_error +if NOT "%~2"=="" goto arg_error +goto arg_end + +:arg_error + +echo Illegal number of parameters. Pass packge type `Conda` or `Wheels`. +exit /b 1 + +:arg_end + +echo "nightly_defaults.bat at %CD% starting at %DATE%" + +set SRC_DIR=%~dp0\.. + +:: NIGHTLIES_FOLDER +:: N.B. this is also defined in cron_start.sh +:: An arbitrary root folder to store all nightlies folders, each of which is a +:: parent level date folder with separate subdirs for logs, wheels, conda +:: packages, etc. This should be kept the same across all scripts called in a +:: cron job, so it only has a default value in the top-most script +:: build_cron.sh to avoid the default values from diverging. +if "%NIGHTLIES_FOLDER%" == "" set "NIGHTLIES_FOLDER=%SRC_DIR%" + +:: NIGHTLIES_DATE +:: N.B. this is also defined in cron_start.sh +:: The date in YYYY_mm_dd format that we are building for. If this is not +:: already set, then this will first try to find the date of the nightlies +:: folder that this builder repo exists in; e.g. if this script exists in +:: some_dir/2019_09_04/builder/cron/ then this will be set to 2019_09_04 (must +:: match YYYY_mm_dd). This is for convenience when debugging/uploading past +:: dates, so that you don't have to set NIGHTLIES_DATE yourself. If a date +:: folder cannot be found in that exact location, then this will default to +:: the current date. + + +if "%NIGHTLIES_DATE%" == "" ( goto date_start ) else ( goto date_end ) + +:date_start + +set "DATE_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyy_MM_dd'" +set "DATE_COMPACT_CMD=Get-Date ([System.TimeZoneInfo]::ConvertTimeFromUtc((Get-Date).ToUniversalTime(), [System.TimeZoneInfo]::FindSystemTimeZoneById('Pacific Standard Time'))) -f 'yyyyMMdd'" + +FOR /F "delims=" %%i IN ('powershell -c "%DATE_CMD%"') DO set NIGHTLIES_DATE=%%i +FOR /F "delims=" %%i IN ('powershell -c "%DATE_COMPACT_CMD%"') DO set NIGHTLIES_DATE_COMPACT=%%i + +:date_end + +if "%NIGHTLIES_DATE_COMPACT%" == "" set NIGHTLIES_DATE_COMPACT=%NIGHTLIES_DATE:~0,4%%NIGHTLIES_DATE:~5,2%%NIGHTLIES_DATE:~8,2% + +:: Used in lots of places as the root dir to store all conda/wheel/manywheel +:: packages as well as logs for the day +set today=%NIGHTLIES_FOLDER%\%NIGHTLIES_DATE% +mkdir "%today%" || ver >nul + + +::############################################################################# +:: Add new configuration variables below this line. 'today' should always be +:: defined ASAP to avoid weird errors +::############################################################################# + + +:: List of people to email when things go wrong. This is passed directly to +:: `mail -t` +:: TODO: Not supported yet +if "%NIGHTLIES_EMAIL_LIST%" == "" set NIGHTLIES_EMAIL_LIST=peterghost86@gmail.com + +:: PYTORCH_CREDENTIALS_FILE +:: A bash file that exports credentials needed to upload to aws and anaconda. +:: Needed variables are PYTORCH_ANACONDA_USERNAME, PYTORCH_ANACONDA_PASSWORD, +:: AWS_ACCESS_KEY_ID, and AWS_SECRET_ACCESS_KEY. Or it can just export the AWS +:: keys and then prepend a logged-in conda installation to the path. +:: TODO: Not supported yet +if "%PYTORCH_CREDENTIALS_FILE%" == "" set PYTORCH_CREDENTIALS_FILE=/c/Users/administrator/nightlies/credentials.sh + +:: Location of the temporary miniconda that is downloaded to install conda-build +:: and aws to upload finished packages TODO this is messy to install this in +:: upload.sh and later use it in upload_logs.sh +if "%CONDA_UPLOADER_INSTALLATION%" == "" set "CONDA_UPLOADER_INSTALLATION=%today%\miniconda" + +:: N.B. BUILDER_REPO and BUILDER_BRANCH are both set in cron_start.sh, as that +:: is the script that actually clones the builder repo that /this/ script is +:: running from. +pushd "%SRC_DIR%\.." +set NIGHTLIES_BUILDER_ROOT=%CD% +popd + +:: The shared pytorch repo to be used by all builds +if "%NIGHTLIES_PYTORCH_ROOT%" == "" set "NIGHTLIES_PYTORCH_ROOT=%today%\vision" + +:: PYTORCH_REPO +:: The Github org/user whose fork of Pytorch to check out (git clone +:: https://github.com//pytorch.git). This will always be cloned +:: fresh to build with. Default is 'pytorch' +if "%PYTORCH_REPO%" == "" set PYTORCH_REPO=pytorch + +:: PYTORCH_BRANCH +:: The branch of Pytorch to checkout for building (git checkout ). +:: This can either be the name of the branch (e.g. git checkout +:: my_branch_name) or can be a git commit (git checkout 4b2674n...). Default +:: is 'latest', which is a special term that signals to pull the last commit +:: before 0:00 midnight on the NIGHTLIES_DATE +if "%PYTORCH_BRANCH%" == "" set PYTORCH_BRANCH=latest + +:: Clone the requested pytorch checkout +if exist "%NIGHTLIES_PYTORCH_ROOT%" ( goto clone_end ) else ( goto clone_start ) + +:clone_start + +git clone --recursive "https://github.com/%PYTORCH_REPO%/vision.git" "%NIGHTLIES_PYTORCH_ROOT%" +pushd "%NIGHTLIES_PYTORCH_ROOT%" + +if "%PYTORCH_BRANCH%" == "latest" ( goto latest_start ) else ( goto latest_end ) + +:latest_start + +:: Switch to the latest commit by 11:59 yesterday +echo PYTORCH_BRANCH is set to latest so I will find the last commit +echo before 0:00 midnight on %NIGHTLIES_DATE% +set git_date=%NIGHTLIES_DATE:_=-% +FOR /F "delims=" %%i IN ('git log --before %git_date% -n 1 "--pretty=%%H"') DO set last_commit=%%i +echo Setting PYTORCH_BRANCH to %last_commit% since that was the last +echo commit before %NIGHTLIES_DATE% +set PYTORCH_BRANCH=%last_commit% + +:latest_end + +git checkout "%PYTORCH_BRANCH%" +git submodule update +popd + +:clone_end + +if "%CUDA_VERSION%" == "cpu" ( + set _DESIRED_CUDA=cpu +) else ( + set _DESIRED_CUDA=cu%CUDA_VERSION% +) + +:: PYTORCH_BUILD_VERSION +:: The actual version string. Used in conda like +:: pytorch-nightly==1.0.0.dev20180908 +:: or in manylinux like +:: torch_nightly-1.0.0.dev20180908-cp27-cp27m-linux_x86_64.whl +if "%TORCHVISION_BUILD_VERSION%" == "" set TORCHVISION_BUILD_VERSION=0.5.0.dev%NIGHTLIES_DATE_COMPACT% + +if "%~1" == "Wheels" ( + if not "%CUDA_VERSION%" == "100" ( + set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%+%_DESIRED_CUDA% + ) +) + +:: PYTORCH_BUILD_NUMBER +:: This is usually the number 1. If more than one build is uploaded for the +:: same version/date, then this can be incremented to 2,3 etc in which case +:: '.post2' will be appended to the version string of the package. This can +:: be set to '0' only if OVERRIDE_PACKAGE_VERSION is being used to bypass +:: all the version string logic in downstream scripts. Since we use the +:: override below, exporting this shouldn't actually matter. +if "%TORCHVISION_BUILD_NUMBER%" == "" set /a TORCHVISION_BUILD_NUMBER=1 +if %TORCHVISION_BUILD_NUMBER% GTR 1 set TORCHVISION_BUILD_VERSION=%TORCHVISION_BUILD_VERSION%%TORCHVISION_BUILD_NUMBER% + +:: The nightly builds use their own versioning logic, so we override whatever +:: logic is in setup.py or other scripts +:: TODO: Not supported yet +set OVERRIDE_PACKAGE_VERSION=%TORCHVISION_BUILD_VERSION% +set BUILD_VERSION=%TORCHVISION_BUILD_VERSION% + +:: Build folder for conda builds to use +if "%TORCH_CONDA_BUILD_FOLDER%" == "" set TORCH_CONDA_BUILD_FOLDER=torchvision + +:: TORCH_PACKAGE_NAME +:: The name of the package to upload. This should probably be pytorch or +:: pytorch-nightly. N.B. that pip will change all '-' to '_' but conda will +:: not. This is dealt with in downstream scripts. +:: TODO: Not supported yet +if "%TORCH_PACKAGE_NAME%" == "" set TORCH_PACKAGE_NAME=torchvision + +:: PIP_UPLOAD_FOLDER should end in a slash. This is to handle it being empty +:: (when uploading to e.g. whl/cpu/) and also to handle nightlies (when +:: uploading to e.g. /whl/nightly/cpu) +:: TODO: Not supported yet +if "%PIP_UPLOAD_FOLDER%" == "" set "PIP_UPLOAD_FOLDER=nightly\" + +:: The location of the binary_sizes dir in s3 is hardcoded into +:: upload_binary_sizes.sh + +:: DAYS_TO_KEEP +:: How many days to keep around for clean.sh. Build folders older than this +:: will be purged at the end of cron jobs. '1' means to keep only the current +:: day. Values less than 1 are not allowed. The default is 5. +:: TODO: Not supported yet +if "%DAYS_TO_KEEP%" == "" set /a DAYS_TO_KEEP=5 +if %DAYS_TO_KEEP% LSS 1 ( + echo DAYS_TO_KEEP cannot be less than 1. + echo A value of 1 means to only keep the build for today + exit /b 1 +) diff --git a/packaging/windows/internal/publish.bat b/packaging/windows/internal/publish.bat new file mode 100644 index 00000000000..6ee9ef853a0 --- /dev/null +++ b/packaging/windows/internal/publish.bat @@ -0,0 +1,85 @@ +@echo off + +set SRC_DIR=%~dp0 +pushd %SRC_DIR% + +if NOT "%CUDA_VERSION%" == "cpu" ( + set PACKAGE_SUFFIX=_cuda%CUDA_VERSION% +) else ( + set PACKAGE_SUFFIX= +) + +if "%PACKAGEFULLNAME%" == "Conda" ( + set PACKAGE=conda +) else ( + set PACKAGE=wheels +) + +set PUBLISH_BRANCH=%PACKAGE%_%DESIRED_PYTHON%%PACKAGE_SUFFIX% + +git clone %ARTIFACT_REPO_URL% -b %PUBLISH_BRANCH% --single-branch >nul 2>&1 + +IF ERRORLEVEL 1 ( + echo Branch %PUBLISH_BRANCH% not exist, falling back to master + set NO_BRANCH=1 + git clone %ARTIFACT_REPO_URL% -b master --single-branch >nul 2>&1 +) + +IF ERRORLEVEL 1 ( + echo Clone failed + goto err +) + +cd pytorch_builder +attrib -s -h -r . /s /d + +:: Empty repo +rd /s /q . || ver >nul + +IF NOT EXIST %PACKAGE% mkdir %PACKAGE% + +xcopy /S /E /Y ..\..\output\*.* %PACKAGE%\ + +git config --global user.name "Azure DevOps" +git config --global user.email peterghost86@gmail.com +git init +git checkout --orphan %PUBLISH_BRANCH% +git remote add origin %ARTIFACT_REPO_URL% +git add . +git commit -m "Update artifacts" + +:push + +if "%RETRY_TIMES%" == "" ( + set /a RETRY_TIMES=10 + set /a SLEEP_TIME=2 +) else ( + set /a RETRY_TIMES=%RETRY_TIMES%-1 + set /a SLEEP_TIME=%SLEEP_TIME%*2 +) + +git push origin %PUBLISH_BRANCH% -f > nul 2>&1 + +IF ERRORLEVEL 1 ( + echo Git push retry times remaining: %RETRY_TIMES% + echo Sleep time: %SLEEP_TIME% seconds + IF %RETRY_TIMES% EQU 0 ( + echo Push failed + goto err + ) + waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul + goto push +) ELSE ( + set RETRY_TIMES= + set SLEEP_TIME= +) + +popd + +exit /b 0 + +:err + +popd + +exit /b 1 diff --git a/packaging/windows/internal/setup.bat b/packaging/windows/internal/setup.bat index fcf10f185c7..d18dfb35023 100644 --- a/packaging/windows/internal/setup.bat +++ b/packaging/windows/internal/setup.bat @@ -28,53 +28,6 @@ if "%CXX%"=="sccache cl" ( sccache --zero-stats ) - -if "%BUILD_PYTHONLESS%" == "" goto pytorch else goto libtorch - -:libtorch -set VARIANT=shared-with-deps - -mkdir libtorch -mkdir libtorch\bin -mkdir libtorch\cmake -mkdir libtorch\include -mkdir libtorch\lib -mkdir libtorch\share -mkdir libtorch\test - -mkdir build -pushd build -python ../tools/build_libtorch.py -popd - -IF ERRORLEVEL 1 exit /b 1 -IF NOT ERRORLEVEL 0 exit /b 1 - -move /Y torch\bin\*.* libtorch\bin\ -move /Y torch\cmake\*.* libtorch\cmake\ -robocopy /move /e torch\include\ libtorch\include\ -move /Y torch\lib\*.* libtorch\lib\ -robocopy /move /e torch\share\ libtorch\share\ -move /Y torch\test\*.* libtorch\test\ - -move /Y libtorch\bin\*.dll libtorch\lib\ - -git rev-parse HEAD > libtorch\build-hash - -IF "%DEBUG%" == "" ( - set LIBTORCH_PREFIX=libtorch-win-%VARIANT% -) ELSE ( - set LIBTORCH_PREFIX=libtorch-win-%VARIANT%-debug -) - -7z a -tzip %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip libtorch\* - -mkdir ..\output\%CUDA_PREFIX% -copy /Y %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip ..\output\%CUDA_PREFIX%\ -copy /Y %LIBTORCH_PREFIX%-%PYTORCH_BUILD_VERSION%.zip ..\output\%CUDA_PREFIX%\%LIBTORCH_PREFIX%-latest.zip - -goto build_end - :pytorch :: This stores in e.g. D:/_work/1/s/windows/output/cpu pip wheel -e . --no-deps --wheel-dir ../output/%CUDA_PREFIX% diff --git a/packaging/windows/internal/test.bat b/packaging/windows/internal/test.bat index 1ad7d2ebeb8..a87fc1a2858 100644 --- a/packaging/windows/internal/test.bat +++ b/packaging/windows/internal/test.bat @@ -8,15 +8,19 @@ set PYTHON_VERSION=%PYTHON_PREFIX:py=cp% if "%BUILD_VISION%" == "" ( pip install future pytest coverage hypothesis protobuf ) ELSE ( - pip install future pytest "pillow>=4.1.1" + pip install future pytest "pillow>=4.1.1" mock ) - -for /F "delims=" %%i in ('where /R %SRC_DIR%\output\%CUDA_PREFIX% %MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i" +for /F "delims=" %%i in ('where /R %SRC_DIR%\output\%CUDA_PREFIX% *%MODULE_NAME%*%PYTHON_VERSION%*.whl') do pip install "%%i" if ERRORLEVEL 1 exit /b 1 -if NOT "%BUILD_VISION%" == "" goto smoke_test_end +if NOT "%BUILD_VISION%" == "" ( + echo Smoke testing imports + python -c "import torchvision" + if ERRORLEVEL 1 exit /b 1 + goto smoke_test_end +) echo Smoke testing imports python -c "import torch" diff --git a/packaging/windows/internal/upload.bat b/packaging/windows/internal/upload.bat new file mode 100644 index 00000000000..0d2946a3fe6 --- /dev/null +++ b/packaging/windows/internal/upload.bat @@ -0,0 +1,96 @@ +@echo off + +IF "%CONDA_UPLOADER_INSTALLATION%" == "" goto precheck_fail +IF "%PYTORCH_FINAL_PACKAGE_DIR%" == "" goto precheck_fail +IF "%today%" == "" goto precheck_fail +IF "%PYTORCH_ANACONDA_USERNAME%" == "" goto precheck_fail +IF "%PYTORCH_ANACONDA_PASSWORD%" == "" goto precheck_fail + +goto precheck_pass + +:precheck_fail + +echo Please run nightly_defaults.bat first. +echo And remember to set `PYTORCH_FINAL_PACKAGE_DIR` +echo Finally, don't forget to set anaconda tokens +exit /b 1 + +:precheck_pass + +pushd %today% + +:: Install anaconda client +set "CONDA_HOME=%CONDA_UPLOADER_INSTALLATION%" +set "tmp_conda=%CONDA_HOME%" +set "miniconda_exe=%CD%\miniconda.exe" +rmdir /s /q "%CONDA_HOME%" +del miniconda.exe +curl -k https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe -o "%miniconda_exe%" +popd + +IF ERRORLEVEL 1 ( + echo Conda download failed + exit /b 1 +) + +call %~dp0\..\..\conda\install_conda.bat + +IF ERRORLEVEL 1 ( + echo Conda installation failed + exit /b 1 +) + +set "ORIG_PATH=%PATH%" +set "PATH=%CONDA_HOME%;%CONDA_HOME%\scripts;%CONDA_HOME%\Library\bin;%PATH%" + +REM conda install -y anaconda-client +pip install git+https://github.com/peterjc123/anaconda-client.git@log_more_meaningfull_errors +IF ERRORLEVEL 1 ( + echo Anaconda client installation failed + exit /b 1 +) + +REM bash -c "yes | anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%"" +anaconda login --username "%PYTORCH_ANACONDA_USERNAME%" --password "%PYTORCH_ANACONDA_PASSWORD%" +IF ERRORLEVEL 1 ( + echo Anaconda client login failed + exit /b 1 +) + +set PYTORCH_FINAL_PACKAGE= +:: Upload all the packages under `PYTORCH_FINAL_PACKAGE_DIR` +FOR /F "delims=" %%i IN ('where /R %PYTORCH_FINAL_PACKAGE_DIR% *vision*.tar.bz2') DO ( + set "PYTORCH_FINAL_PACKAGE=%%i" +) + +IF "%PYTORCH_FINAL_PACKAGE%" == "" ( + echo No package to upload + exit /b 0 +) + +:upload + +if "%RETRY_TIMES%" == "" ( + set /a RETRY_TIMES=10 + set /a SLEEP_TIME=2 +) else ( + set /a RETRY_TIMES=%RETRY_TIMES%-1 + set /a SLEEP_TIME=%SLEEP_TIME%*2 +) + +echo Uploading %PYTORCH_FINAL_PACKAGE% to Anaconda Cloud +anaconda upload "%PYTORCH_FINAL_PACKAGE%" -u pytorch-nightly --label main --force --no-progress + +IF ERRORLEVEL 1 ( + echo Anaconda upload retry times remaining: %RETRY_TIMES% + echo Sleep time: %SLEEP_TIME% seconds + IF %RETRY_TIMES% EQU 0 ( + echo Upload failed + exit /b 1 + ) + waitfor SomethingThatIsNeverHappening /t %SLEEP_TIME% 2>nul || ver >nul + goto upload +) ELSE ( + set RETRY_TIMES= + set SLEEP_TIME= +) diff --git a/packaging/windows/internal/vs_install.bat b/packaging/windows/internal/vs_install.bat index 624227f0be0..e6589092372 100644 --- a/packaging/windows/internal/vs_install.bat +++ b/packaging/windows/internal/vs_install.bat @@ -1,23 +1,23 @@ @echo off set VS_DOWNLOAD_LINK=https://aka.ms/vs/15/release/vs_buildtools.exe -IF "%VS_LATEST%" == "1" ( - set VS_INSTALL_ARGS= --nocache --norestart --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools - set VSDEVCMD_ARGS= -) ELSE ( - set VS_INSTALL_ARGS=--nocache --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools ^ - --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^ - --add Microsoft.Component.MSBuild ^ - --add Microsoft.VisualStudio.Component.Roslyn.Compiler ^ - --add Microsoft.VisualStudio.Component.TextTemplating ^ - --add Microsoft.VisualStudio.Component.VC.CoreIde ^ - --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest ^ - --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core ^ - --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^ - --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^ - --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 - set VSDEVCMD_ARGS=-vcvars_ver=14.11 -) +REM IF "%VS_LATEST%" == "1" ( +REM set VS_INSTALL_ARGS= --nocache --norestart --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools +REM set VSDEVCMD_ARGS= +REM ) ELSE ( +set VS_INSTALL_ARGS=--nocache --quiet --wait --add Microsoft.VisualStudio.Workload.VCTools ^ + --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^ + --add Microsoft.Component.MSBuild ^ + --add Microsoft.VisualStudio.Component.Roslyn.Compiler ^ + --add Microsoft.VisualStudio.Component.TextTemplating ^ + --add Microsoft.VisualStudio.Component.VC.CoreIde ^ + --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest ^ + --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core ^ + --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^ + --add Microsoft.VisualStudio.Component.VC.Tools.14.11 ^ + --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81 +set VSDEVCMD_ARGS=-vcvars_ver=14.11 +REM ) curl -k -L %VS_DOWNLOAD_LINK% --output vs_installer.exe if errorlevel 1 exit /b 1 diff --git a/packaging/windows/cuda90.bat b/packaging/windows/old/cuda90.bat similarity index 100% rename from packaging/windows/cuda90.bat rename to packaging/windows/old/cuda90.bat diff --git a/packaging/windows/templates/auth_task.yml b/packaging/windows/templates/auth_task.yml new file mode 100644 index 00000000000..ece66412ff4 --- /dev/null +++ b/packaging/windows/templates/auth_task.yml @@ -0,0 +1,17 @@ +jobs: +- job: 'VSTS_Auth_Task' + timeoutInMinutes: 5 + cancelTimeoutInMinutes: 5 + variables: + - group: 'peterjc-vsts-token' + + pool: + vmImage: 'win1803' + + steps: + - checkout: self + clean: true + + - template: vsts_auth.yml + parameters: + auth: $(vsts_auth) diff --git a/packaging/windows/templates/build_conda.yml b/packaging/windows/templates/build_conda.yml new file mode 100644 index 00000000000..2d88271ad33 --- /dev/null +++ b/packaging/windows/templates/build_conda.yml @@ -0,0 +1,15 @@ +parameters: + msagent: false + +steps: +- bash: 'find . -name "*.sh" -exec dos2unix {} +' + displayName: Replace file endings + +- script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' + displayName: 'Create final package directory' + +- bash: './packaging/conda/build_vision.sh $CUDA_VERSION $TORCHVISION_BUILD_VERSION $TORCHVISION_BUILD_NUMBER' + displayName: Build + env: + ${{ if eq(parameters.msagent, 'true') }}: + MAX_JOBS: 2 diff --git a/packaging/windows/templates/build_task.yml b/packaging/windows/templates/build_task.yml new file mode 100644 index 00000000000..8ecbf93abfa --- /dev/null +++ b/packaging/windows/templates/build_task.yml @@ -0,0 +1,140 @@ +parameters: + package: '' + spec: '' + jobDesc: '' + packageDesc: '' + msagent: true + cpuEnabled: true + cudaEnabled: true + condaEnabled: true + wheelsEnabled: true + override: false + +jobs: +- job: 'Windows_${{ parameters.spec }}_${{ parameters.package }}_Build' + timeoutInMinutes: 60 + cancelTimeoutInMinutes: 5 + condition: > + or(and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CPU'), + eq('${{ parameters.condaEnabled }}', 'true'), eq('${{ parameters.cpuEnabled }}', 'true')), + and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CPU'), + eq('${{ parameters.wheelsEnabled }}', 'true'), eq('${{ parameters.cpuEnabled }}', 'true')), + and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CUDA'), + eq('${{ parameters.condaEnabled }}', 'true'), eq('${{ parameters.cudaEnabled }}', 'true')), + and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CUDA'), + eq('${{ parameters.wheelsEnabled }}', 'true'), eq('${{ parameters.cudaEnabled }}', 'true'))) + variables: + - ${{ if eq(parameters.override, 'true') }}: + - name: TORCHVISION_BUILD_NUMBER + value: 1 + - name: PYTORCH_REPO + value: 'pytorch' + - name: PYTORCH_BRANCH + value: 'v0.4.0' + - ${{ if eq(parameters.msagent, 'true') }}: + - name: USE_SCCACHE + value: 0 + - ${{ if eq(parameters.msagent, 'false') }}: + - name: USE_SCCACHE + value: 1 + - ${{ if eq(parameters.package, 'Conda') }}: + - group: peterjc_anaconda_token + - name: PYTORCH_FINAL_PACKAGE_DIR + value: '$(Build.Repository.LocalPath)\packaging\windows\output' + + strategy: + maxParallel: 10 + matrix: + ${{ if eq(parameters.spec, 'CPU') }}: + PY3.5: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: cpu + PY3.6: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: cpu + PY3.7: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: cpu + ${{ if ne(parameters.spec, 'CPU') }}: + # PY3.5_92: + # DESIRED_PYTHON: 3.5 + # CUDA_VERSION: 92 + # PY3.6_92: + # DESIRED_PYTHON: 3.6 + # CUDA_VERSION: 92 + # PY3.7_92: + # DESIRED_PYTHON: 3.7 + # CUDA_VERSION: 92 + PY3.5_100: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 100 + PY3.6_100: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 100 + PY3.7_100: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 100 + + pool: + ${{ if eq(parameters.msagent, 'true') }}: + vmImage: 'win1803' + ${{ if eq(parameters.msagent, 'false') }}: + name: 'release' + + steps: + - checkout: self + clean: true + + - template: setup_env_for_msagent.yml + parameters: + msagent: ${{ parameters.msagent }} + + # - ${{ if and(eq(parameters.override, 'true'), eq(parameters.package, 'Wheels')) }}: + # - template: override_pytorch_version.yml + + - template: setup_nightly_variables.yml + parameters: + package: ${{ parameters.package }} + + - ${{ if eq(parameters.package, 'Wheels') }}: + - template: build_wheels.yml + parameters: + msagent: ${{ parameters.msagent }} + + - ${{ if eq(parameters.package, 'Conda') }}: + - template: build_conda.yml + parameters: + msagent: ${{ parameters.msagent }} + + - ${{ if or(eq(parameters.package, 'Wheels'), eq(parameters.package, 'Conda')) }}: + - template: publish_test_results.yml + parameters: + msagent: ${{ parameters.msagent }} + + # If you want to upload binaries to S3 & Anaconda Cloud, please uncomment this section. + - ${{ if and(eq(parameters.package, 'Wheels'), eq(parameters.spec, 'CPU')) }}: + - template: upload_to_s3.yml + parameters: + cuVer: '$(CUDA_VERSION)' + cudaVer: '$(CUDA_VERSION)' + + - ${{ if and(eq(parameters.package, 'Wheels'), ne(parameters.spec, 'CPU')) }}: + - template: upload_to_s3.yml + parameters: + cuVer: 'cu$(CUDA_VERSION)' + cudaVer: 'cuda$(CUDA_VERSION)' + + - ${{ if eq(parameters.package, 'Conda') }}: + - template: upload_to_conda.yml + parameters: + user: $(peterjc_conda_username) + pass: $(peterjc_conda_password) + + # If you want to upload binaries to Azure Git, please uncomment this section. + # - ${{ if or(eq(parameters.package, 'Wheels'), eq(parameters.package, 'Conda')) }}: + # - template: publish_test_results.yml + # parameters: + # msagent: ${{ parameters.msagent }} + # - template: publish_packages.yml + # parameters: + # package: ${{ parameters.package }} diff --git a/packaging/windows/templates/build_wheels.yml b/packaging/windows/templates/build_wheels.yml new file mode 100644 index 00000000000..05c5712e334 --- /dev/null +++ b/packaging/windows/templates/build_wheels.yml @@ -0,0 +1,9 @@ +parameters: + msagent: false + +steps: +- script: 'call packaging/windows/build_vision.bat %CUDA_VERSION% %TORCHVISION_BUILD_VERSION% %TORCHVISION_BUILD_NUMBER%' + displayName: Build + env: + ${{ if eq(parameters.msagent, 'true') }}: + MAX_JOBS: 2 diff --git a/packaging/windows/templates/linux_build_task.yml b/packaging/windows/templates/linux_build_task.yml new file mode 100644 index 00000000000..0b32892791a --- /dev/null +++ b/packaging/windows/templates/linux_build_task.yml @@ -0,0 +1,38 @@ +parameters: + msagent: true + enabled: false + +jobs: +- job: 'Linux_CPU_Conda_Build' + timeoutInMinutes: 0 + cancelTimeoutInMinutes: 5 + condition: ${{ eq(parameters.enabled, 'true') }} + variables: + CUDA_VERSION: cpu + TORCH_CONDA_BUILD_FOLDER: pytorch-nightly + PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)/output' + + strategy: + maxParallel: 10 + matrix: + PY3.5: + DESIRED_PYTHON: 3.5 + + pool: + vmImage: 'ubuntu-16.04' + + steps: + - checkout: self + clean: true + + - script: 'sudo apt-get install p7zip-full' + displayName: 'Install 7Zip' + + - task: CondaEnvironment@1 + displayName: 'Install conda-build' + inputs: + packageSpecs: 'conda-build' + + - template: build_conda.yml + parameters: + msagent: ${{ parameters.msagent }} diff --git a/packaging/windows/templates/override_pytorch_version.yml b/packaging/windows/templates/override_pytorch_version.yml new file mode 100644 index 00000000000..8af93ae43a4 --- /dev/null +++ b/packaging/windows/templates/override_pytorch_version.yml @@ -0,0 +1,6 @@ +steps: +- script: 'windows/internal/override_pytorch_version.bat' + displayName: 'Override PyTorch Build Version for Wheels' + +- script: 'echo $(PYTORCH_BUILD_VERSION)' + displayName: 'Show PyTorch Build Version' diff --git a/packaging/windows/templates/publish_packages.yml b/packaging/windows/templates/publish_packages.yml new file mode 100644 index 00000000000..51ce8247bf7 --- /dev/null +++ b/packaging/windows/templates/publish_packages.yml @@ -0,0 +1,8 @@ +parameters: + package: '' + +steps: +- script: 'packaging/windows/internal/publish.bat' + displayName: 'Upload packages to Azure DevOps Repo' + env: + PACKAGEFULLNAME: ${{ parameters.package }} diff --git a/packaging/windows/templates/publish_test_results.yml b/packaging/windows/templates/publish_test_results.yml new file mode 100644 index 00000000000..1e0dc0215d3 --- /dev/null +++ b/packaging/windows/templates/publish_test_results.yml @@ -0,0 +1,6 @@ +steps: +- task: PublishTestResults@2 # No test results to publish + inputs: + testResultsFiles: 'windows/pytorch/test/**/*.xml' + testRunTitle: 'Publish test results' + enabled: false diff --git a/packaging/windows/templates/setup_env_for_msagent.yml b/packaging/windows/templates/setup_env_for_msagent.yml new file mode 100644 index 00000000000..377734fa3db --- /dev/null +++ b/packaging/windows/templates/setup_env_for_msagent.yml @@ -0,0 +1,25 @@ +parameters: + msagent: false + +steps: +- ${{ if eq(parameters.msagent, 'true') }}: + - task: BatchScript@1 + displayName: 'Install 7Zip & cURL' + inputs: + filename: 'packaging/windows/internal/dep_install.bat' + + modifyEnvironment: true + + - task: BatchScript@1 + displayName: 'Install Visual Studio 2017' + inputs: + filename: 'packaging/windows/internal/vs_install.bat' + + modifyEnvironment: true + + - task: BatchScript@1 + displayName: 'Install CUDA' + inputs: + filename: 'packaging/windows/internal/cuda_install.bat' + + modifyEnvironment: true diff --git a/packaging/windows/templates/setup_nightly_variables.yml b/packaging/windows/templates/setup_nightly_variables.yml new file mode 100644 index 00000000000..94b2fe934ce --- /dev/null +++ b/packaging/windows/templates/setup_nightly_variables.yml @@ -0,0 +1,11 @@ +parameters: + package: '' + +steps: +- task: BatchScript@1 + displayName: 'Setup nightly variables' + inputs: + filename: 'packaging/windows/internal/nightly_defaults.bat' + arguments: ${{ parameters.package }} + + modifyEnvironment: true diff --git a/packaging/windows/templates/upload_to_conda.yml b/packaging/windows/templates/upload_to_conda.yml new file mode 100644 index 00000000000..dc172bcf878 --- /dev/null +++ b/packaging/windows/templates/upload_to_conda.yml @@ -0,0 +1,10 @@ +parameters: + user: '' + pass: '' + +steps: +- script: 'call packaging/windows/internal/upload.bat' + displayName: 'Upload packages to Anaconda Cloud' + env: + PYTORCH_ANACONDA_USERNAME: ${{ parameters.user }} + PYTORCH_ANACONDA_PASSWORD: ${{ parameters.pass }} diff --git a/packaging/windows/templates/upload_to_s3.yml b/packaging/windows/templates/upload_to_s3.yml new file mode 100644 index 00000000000..a31bcb15ae1 --- /dev/null +++ b/packaging/windows/templates/upload_to_s3.yml @@ -0,0 +1,15 @@ +parameters: + cuVer: '' + cudaVer: '' + +steps: +- task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1 + displayName: 'Upload ${{ parameters.cuVer }} wheel to S3' + inputs: + awsCredentials: 'Pytorch S3 bucket' + bucketName: 'pytorch' + sourceFolder: 'packaging/windows/output/${{ parameters.cudaVer }}' + globExpressions: '*.whl' + targetFolder: 'whl/nightly/${{ parameters.cuVer }}/' + filesAcl: 'public-read' + flattenFolders: 'true' diff --git a/packaging/windows/templates/vsts_auth.yml b/packaging/windows/templates/vsts_auth.yml new file mode 100644 index 00000000000..fde767d7f12 --- /dev/null +++ b/packaging/windows/templates/vsts_auth.yml @@ -0,0 +1,8 @@ +parameters: + auth: '' + +steps: +- script: 'call packaging/windows/internal/auth.bat' + displayName: 'Sign in to Azure Pipelines' + env: + VSTS_AUTH: ${{ parameters.auth }}