Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update on "Deprecate torch.chain_matmul in favor of torch.linalg.mult…
…i_dot" Differential Revision: [D27406282](https://our.internmc.facebook.com/intern/diff/D27406282) [ghstack-poisoned]
- Loading branch information
Showing
87 changed files
with
2,626 additions
and
801 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
# PyTorch CI Builds Pipeline on Azure DevOps | ||
# | ||
# This pipeline: | ||
# 1) builds PyTorch on select configurations | ||
# 2) runs only TestTorch unit tests. | ||
|
||
stages: | ||
- stage: 'Build' | ||
displayName: 'Build PyTorch' | ||
jobs: | ||
- template: job_templates/build-verify-publish-template-unix.yml | ||
parameters: | ||
name: ubuntu_1804_CPU_docker | ||
pool: 'PyTorch-Linux-CPU' | ||
container_endpoint: pytorchms.azurecr.io | ||
build_stage: True | ||
is_ci_build: True | ||
os: ubuntu | ||
cuda: cpu | ||
customMatrixes: | ||
Py_38: | ||
configuration: ubuntu_1804_py_38_cpu | ||
container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cpu_dev | ||
|
||
- template: job_templates/build-verify-publish-template-unix.yml | ||
parameters: | ||
name: ubuntu_1804_GPU_docker | ||
pool: 'PyTorch-Linux-GPU' | ||
container_endpoint: pytorchms.azurecr.io | ||
build_stage: True | ||
is_ci_build: True | ||
os: ubuntu | ||
cuda: gpu | ||
customMatrixes: | ||
Py_39_CUDA_112_cuDNN_810: | ||
configuration: ubuntu_1804_py_39_cuda_112_cudnn_810 | ||
container_image: pytorchms.azurecr.io/ubuntu_1804_py_39_cuda_112_cudnn_8_dev | ||
CUDA_VERSION: 112 | ||
|
||
- template: job_templates/build-verify-publish-template-win.yml | ||
parameters: | ||
name: windows_2019_CPU | ||
pool: 'PyTorch-Win-CPU' | ||
build_stage: True | ||
is_ci_build: True | ||
os: windows | ||
cuda: cpu | ||
customMatrixes: | ||
Py_37: | ||
configuration: windows_2019_py_37_cpu | ||
|
||
- template: job_templates/build-verify-publish-template-win.yml | ||
parameters: | ||
name: windows_2019_GPU | ||
pool: 'PyTorch-Win-GPU' | ||
build_stage: True | ||
is_ci_build: True | ||
os: windows | ||
cuda: gpu | ||
customMatrixes: | ||
Py_38_CUDA_102_cuDNN_765: | ||
configuration: windows_2019_py_38_cuda_102_cudnn_765 | ||
CUDA_VERSION: 102 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
# PyTorch Daily Builds Pipeline on Azure DevOps | ||
# | ||
# This pipeline: | ||
# 1) builds PyTorch on all available configurations | ||
# 2) runs all PyTorch unit tests | ||
|
||
stages: | ||
- stage: 'BuildTest' | ||
displayName: 'Build and Test PyTorch' | ||
jobs: | ||
- template: job_templates/build-verify-publish-template-unix.yml | ||
parameters: | ||
name: ubuntu_1804_CPU_docker | ||
pool: 'PyTorch-Linux-CPU' | ||
container_endpoint: pytorchms.azurecr.io | ||
build_stage: True | ||
is_daily_build: True | ||
os: ubuntu | ||
cuda: cpu | ||
customMatrixes: | ||
Py_38: | ||
configuration: ubuntu_1804_py_38_cpu | ||
container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cpu_dev | ||
Py_37: | ||
configuration: ubuntu_1804_py_37_cpu | ||
container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cpu_dev | ||
|
||
- template: job_templates/build-verify-publish-template-unix.yml | ||
parameters: | ||
name: ubuntu_1804_GPU_docker | ||
pool: 'PyTorch-Linux-GPU' | ||
container_endpoint: pytorchms.azurecr.io | ||
build_stage: True | ||
is_daily_build: True | ||
os: ubuntu | ||
cuda: gpu | ||
customMatrixes: | ||
Py_39_CUDA_112_cuDNN_810: | ||
configuration: ubuntu_1804_py_39_cuda_112_cudnn_810 | ||
container_image: pytorchms.azurecr.io/ubuntu_1804_py_39_cuda_112_cudnn_8_dev | ||
CUDA_VERSION: 112 | ||
Py_38_CUDA_102_cuDNN_810: | ||
configuration: ubuntu_1804_py_38_cuda_102_cudnn_810 | ||
container_image: pytorchms.azurecr.io/ubuntu_1804_py_38_cuda_102_cudnn_8_dev | ||
CUDA_VERSION: 102 | ||
Py_37_CUDA_101_cuDNN_765: | ||
configuration: ubuntu_1804_py_37_cuda_101_cudnn_765 | ||
container_image: pytorchms.azurecr.io/ubuntu_1804_py_37_cuda_101_cudnn_7_dev | ||
CUDA_VERSION: 101 | ||
|
||
- template: job_templates/build-verify-publish-template-win.yml | ||
parameters: | ||
name: windows_2019_CPU | ||
pool: 'PyTorch-Win-CPU' | ||
build_stage: True | ||
is_daily_build: True | ||
os: windows | ||
cuda: cpu | ||
customMatrixes: | ||
Py_38: | ||
configuration: windows_2019_py_38_cpu | ||
Py_37: | ||
configuration: windows_2019_py_37_cpu | ||
|
||
- template: job_templates/build-verify-publish-template-win.yml | ||
parameters: | ||
name: windows_2019_GPU | ||
pool: 'PyTorch-Win-GPU' | ||
build_stage: True | ||
is_daily_build: True | ||
os: windows | ||
cuda: gpu | ||
customMatrixes: | ||
Py_39_CUDA_112_cuDNN_810: | ||
configuration: windows_2019_py_39_cuda_112_cudnn_810 | ||
CUDA_VERSION: 112 | ||
Py_38_CUDA_102_cuDNN_765: | ||
configuration: windows_2019_py_38_cuda_102_cudnn_765 | ||
CUDA_VERSION: 102 | ||
Py_37_CUDA_101_cuDNN_764: | ||
configuration: windows_2019_py_37_cuda_101_cudnn_764 | ||
CUDA_VERSION: 101 |
134 changes: 134 additions & 0 deletions
134
.azure_pipelines/job_templates/build-verify-publish-template-unix.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,134 @@ | ||
# PyTorch build steps template with Unix images Azure DevOps Instances | ||
# | ||
# This build depends on 3 parameters set as environment variables in the pipeline: | ||
# - AZURE_DEVOPS_CLI_PAT: Secret var for authenticating to Azure DevOps | ||
# - AZURE_DEVOPS_ARTIFACTS_ORGANIZATION: Azure Artifacts Organization name to publish artifacts | ||
# - AZURE_DEVOPS_ARTIFACTS_PROJECT: Azure Artifacts Project name to publish artifacts | ||
|
||
parameters: | ||
name: '' | ||
pool: '' | ||
container_endpoint: '' | ||
os: '' | ||
cuda: '' | ||
is_ci_build: False | ||
is_official_build: False | ||
is_daily_build: False | ||
build_stage: False | ||
verify_stage: False | ||
publish_stage: False | ||
customMatrixes: '' | ||
|
||
jobs: | ||
- job: ${{parameters.name}} | ||
timeoutInMinutes: 300 | ||
strategy: | ||
matrix: | ||
${{ insert }}: ${{parameters.customMatrixes}} | ||
pool: | ||
name: ${{ parameters.pool}} | ||
variables: | ||
DECODE_PERCENTS: false | ||
container: | ||
image: $[variables['container_image']] | ||
endpoint: ${{parameters.container_endpoint}} | ||
|
||
steps: | ||
# Build stage | ||
- ${{ if eq(parameters.build_stage, 'True') }}: | ||
# Set up environment variables for specific pipeline build | ||
- template: set-environment-variables.yml | ||
parameters: | ||
os: ${{ parameters.os}} | ||
cuda: ${{ parameters.cuda}} | ||
is_official_build: ${{ parameters.is_official_build}} | ||
|
||
# Sync and update PyTorch submodules | ||
- bash: git submodule update --init --recursive | ||
displayName: Update PyTorch submodules | ||
|
||
# Build PyTorch and run unit tests - no packaging | ||
- ${{ if or(eq(parameters.is_ci_build, 'True'), eq(parameters.is_daily_build, 'True')) }}: | ||
# Build PyTorch from source in develop mode | ||
- bash: python setup.py develop | ||
displayName: Build PyTorch from source | ||
|
||
- ${{ if eq(parameters.is_ci_build, 'True') }}: | ||
# Run TestTorch unit tests to demonstrate successful PyTorch build | ||
- bash: python test/test_torch.py TestTorch | ||
displayName: Run TestTorch unit tests | ||
|
||
- ${{ if eq(parameters.is_daily_build, 'True') }}: | ||
# Run all unit tests to demonstrate successful PyTorch build | ||
- bash: python test/run_test.py --continue-through-error --exclude-jit-executor --verbose | ||
displayName: Run all unit tests | ||
|
||
# Run ComponentGovernance | ||
- task: ComponentGovernanceComponentDetection@0 | ||
inputs: | ||
scanType: 'Register' | ||
verbosity: 'Verbose' | ||
alertWarningLevel: 'High' | ||
|
||
# Build PyTorch and produce artifacts for verification stage | ||
- ${{ if eq(parameters.is_official_build, 'True') }}: | ||
# Build PyTorch from source in install mode and exclude test binaries | ||
- bash: python setup.py install | ||
displayName: Build PyTorch from source without test binaries | ||
|
||
# Package PyTorch Wheel | ||
- bash: python setup.py bdist_wheel | ||
displayName: Package PyTorch Wheel | ||
|
||
# Publish PyTorch Wheel | ||
- task: PublishPipelineArtifact@1 | ||
inputs: | ||
targetPath: $(Build.SourcesDirectory)/dist/ | ||
artifactName: Build_$(Build.BuildNumber)_$(configuration) | ||
displayName: Publish PyTorch Wheel to Pipeline Artifacts | ||
|
||
# Verification stage | ||
- ${{ if eq(parameters.verify_stage, 'True') }}: | ||
# Download PyTorch Wheel | ||
- task: DownloadPipelineArtifact@2 | ||
inputs: | ||
artifact: Build_$(Build.BuildNumber)_$(configuration) | ||
path: $(Build.SourcesDirectory)/verify | ||
displayName: Download PyTorch Wheel | ||
|
||
# Install PyTorch Wheel on Windows | ||
- bash: python -m pip install $(Build.SourcesDirectory)/verify/torch*linux*.whl | ||
displayName: Install PyTorch Wheel | ||
|
||
# Ensure PyTorch installed correctly from produced wheel | ||
- bash: | | ||
cd $(Build.SourcesDirectory)/verify | ||
python -c "import torch; print('Installed Torch version: ' + torch.__version__)" | ||
displayName: Check PyTorch correctly installed from wheel | ||
# Publishing stage | ||
- ${{ if eq(parameters.publish_stage, 'True') }}: | ||
# Download PyTorch Wheel | ||
- task: DownloadPipelineArtifact@2 | ||
inputs: | ||
artifact: Build_$(Build.BuildNumber)_$(configuration) | ||
path: $(Build.SourcesDirectory)/publish | ||
displayName: Download PyTorch Wheel | ||
|
||
# Publish wheel to Azure Artifacts | ||
# The flag continueOnError=true is needed as the artifact to be published | ||
# may already exist, because the artifact is differentiated based on the | ||
# last commit date. | ||
- bash: | | ||
export TORCH_VERSION=$(head -c 5 ./version.txt) | ||
export LAST_COMMIT=$(git rev-parse --short HEAD) | ||
export LAST_COMMIT_DATE=$(git log -1 --pretty=%ad --date=format:%Y%m%d) | ||
cd $(Build.SourcesDirectory)/publish | ||
export TORCH_WHEEL=$(echo torch*linux*whl) | ||
az extension add -n azure-devops | ||
echo $ADOTOKEN | az devops login | ||
az artifacts universal publish --organization $AZURE_DEVOPS_ARTIFACTS_ORGANIZATION --project $AZURE_DEVOPS_ARTIFACTS_PROJECT --scope project --feed "PyTorch" --name $TORCH_WHEEL --description "PyTorch Official Build Artifact" --version $TORCH_VERSION-$LAST_COMMIT_DATE-$LAST_COMMIT --path . | ||
env: | ||
ADOTOKEN: $(AZURE_DEVOPS_CLI_PAT) | ||
continueOnError: true | ||
displayName: Upload PyTorch Official Build package to Azure Artifacts |
Oops, something went wrong.