Merge remote-tracking branch 'upstream/master' into fast_gelu

pytorch · Jul 13, 2021 · 7d2cd5a · 7d2cd5a
2 parents 57a292c + d520406
commit 7d2cd5a
Show file tree

Hide file tree

Showing 244 changed files with 6,837 additions and 2,928 deletions.
diff --git a/.circleci/cimodel/data/windows_build_definitions.py b/.circleci/cimodel/data/windows_build_definitions.py
@@ -78,6 +78,7 @@ def gen_tree(self):
             props_dict = {
                 "build_environment": build_environment_string,
                 "python_version": miniutils.quote(python_version),
+                "vs_version": miniutils.quote("16.8.6"),
                 "vc_version": miniutils.quote(self.vscode_spec.dotted_version()),
                 "vc_year": miniutils.quote(str(self.vscode_spec.year)),
                 "vc_product": self.vscode_spec.get_product(),

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -357,6 +357,9 @@ pytorch_windows_params: &pytorch_windows_params
     python_version:
       type: string
       default: "3.8"
+    vs_version:
+      type: string
+      default: "16.8.6"
     vc_version:
       type: string
       default: "14.16"
@@ -374,6 +377,7 @@ pytorch_windows_params: &pytorch_windows_params
     SCCACHE_BUCKET: "ossci-compiler-cache"
     CUDA_VERSION: <<parameters.cuda_version>>
     PYTHON_VERSION: <<parameters.python_version>>
+    VS_VERSION: <<parameters.vs_version>>
     VC_VERSION: <<parameters.vc_version>>
     VC_YEAR: <<parameters.vc_year>>
     VC_PRODUCT: <<parameters.vc_product>>
@@ -689,8 +693,7 @@ jobs:
           export JOB_BASE_NAME="$CIRCLE_JOB"
           export CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID"
           cd workspace
-          export PYTHONPATH="\${PWD}"
-          python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test
+          python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
           EOL
           echo "(cat docker_commands.sh | docker exec -u jenkins -e LANG=C.UTF-8 -i "$id" bash) 2>&1" > command.sh
           unbuffer bash command.sh | ts
@@ -720,6 +723,9 @@ jobs:
       python_version:
         type: string
         default: "3.8"
+      vs_version:
+        type: string
+        default: "16.8.6"
       vc_version:
         type: string
         default: "14.16"
@@ -787,6 +793,9 @@ jobs:
       python_version:
         type: string
         default: "3.8"
+      vs_version:
+        type: string
+        default: "16.8.6"
       vc_version:
         type: string
         default: "14.16"
@@ -842,9 +851,8 @@ jobs:
             set -ex
             export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_WIN_BUILD_V1}
             export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_WIN_BUILD_V1}
-            export PYTHONPATH="$PWD"
             pip install typing_extensions boto3
-            python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test
+            python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
           when: always
       - store_test_results:
           path: test/test-reports
@@ -1454,12 +1462,11 @@ jobs:
             set -ex
             source /Users/distiller/workspace/miniconda3/bin/activate
             pip install boto3
-            export PYTHONPATH="$PWD"
 
             # Using the same IAM user to write stats to our OSS bucket
             export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
             export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
-            python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test
+            python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
           when: always
       - store_test_results:
           path: test/test-reports
@@ -7624,6 +7631,7 @@ workflows:
           vc_product: BuildTools
           vc_version: ""
           vc_year: "2019"
+          vs_version: "16.8.6"
       - pytorch_windows_test:
           build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
           cuda_version: "10.1"
@@ -7642,6 +7650,7 @@ workflows:
           vc_product: BuildTools
           vc_version: ""
           vc_year: "2019"
+          vs_version: "16.8.6"
       - update_s3_htmls:
           context: org-member
           filters:
@@ -9182,6 +9191,7 @@ workflows:
           vc_product: BuildTools
           vc_version: ""
           vc_year: "2019"
+          vs_version: "16.8.6"
       - pytorch_windows_test:
           build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
           cuda_version: "10.1"
@@ -9194,6 +9204,7 @@ workflows:
           vc_product: BuildTools
           vc_version: ""
           vc_year: "2019"
+          vs_version: "16.8.6"
     when: << pipeline.parameters.run_master_build >>
   slow_gradcheck_build:
     jobs:
@@ -9307,6 +9318,7 @@ workflows:
           python_version: "3.8"
           use_cuda: "1"
           vc_product: BuildTools
+          vs_version: "16.8.6"
           vc_version: "14.28.29333"
           vc_year: "2019"
           filters:
@@ -9325,6 +9337,7 @@ workflows:
           test_name: pytorch-windows-test1
           use_cuda: "1"
           vc_product: BuildTools
+          vs_version: "16.8.6"
           vc_version: "14.28.29333"
           vc_year: "2019"
           filters:
@@ -9343,6 +9356,7 @@ workflows:
           test_name: pytorch-windows-test2
           use_cuda: "1"
           vc_product: BuildTools
+          vs_version: "16.8.6"
           vc_version: "14.28.29333"
           vc_year: "2019"
           filters:

diff --git a/.circleci/scripts/vs_install.ps1 b/.circleci/scripts/vs_install.ps1
@@ -1,8 +1,8 @@
 # https://developercommunity.visualstudio.com/t/install-specific-version-of-vs-component/1142479
-# https://docs.microsoft.com/en-us/visualstudio/releases/2019/history#release-dates-and-build-numbers
+# Where to find the links: https://docs.microsoft.com/en-us/visualstudio/releases/2019/history#release-dates-and-build-numbers
 
-# 16.8.6 BuildTools
-$VS_DOWNLOAD_LINK = "https://s3.amazonaws.com/ossci-windows/vs16.8.6_BuildTools.exe"
+# BuildTools from S3
+$VS_DOWNLOAD_LINK = "https://s3.amazonaws.com/ossci-windows/vs${env:VS_VERSION}_BuildTools.exe"
 $COLLECT_DOWNLOAD_LINK = "https://aka.ms/vscollect.exe"
 $VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
                                                      "--add Microsoft.Component.MSBuild",
@@ -18,39 +18,48 @@ if (${env:INSTALL_WINDOWS_SDK} -eq "1") {
     $VS_INSTALL_ARGS += "--add Microsoft.VisualStudio.Component.Windows10SDK.19041"
 }
 
+if (Test-Path "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe") {
+    $VS_VERSION_major = [int] ${env:VS_VERSION}.split(".")[0]
+    $existingPath = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -products "Microsoft.VisualStudio.Product.BuildTools" -version "[${env:VS_VERSION}, ${env:VS_VERSION_major + 1})" -property installationPath
+    if (($existingPath -ne $null) -and (!${env:CIRCLECI})) {
+        echo "Found correctly versioned existing BuildTools installation in $existingPath"
+        exit 0
+    }
+    $pathToRemove = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -products "Microsoft.VisualStudio.Product.BuildTools" -property installationPath
+}
+
+echo "Downloading VS installer from S3."
 curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
 if ($LASTEXITCODE -ne 0) {
-    echo "Download of the VS 2019 Version 16.8.6 installer failed"
+    echo "Download of the VS 2019 Version ${env:VS_VERSION} installer failed"
     exit 1
 }
 
-if (Test-Path "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe") {
-    $existingPath = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -products "Microsoft.VisualStudio.Product.BuildTools" -version "[16, 17)" -property installationPath
-    if ($existingPath -ne $null) {
-        echo "Found existing BuildTools installation in $existingPath"
-        $VS_UNINSTALL_ARGS = @("uninstall", "--installPath", "`"$existingPath`"", "--quiet","--wait")
-        $process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_UNINSTALL_ARGS -NoNewWindow -Wait -PassThru
-        $exitCode = $process.ExitCode
-        if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
-            echo "Original BuildTools uninstall failed with code $exitCode"
-            exit 1
-        }
-        echo "Original BuildTools uninstalled"
+if ($pathToRemove -ne $null) {
+    echo "Uninstalling $pathToRemove."
+    $VS_UNINSTALL_ARGS = @("uninstall", "--installPath", "`"$pathToRemove`"", "--quiet","--wait")
+    $process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_UNINSTALL_ARGS -NoNewWindow -Wait -PassThru
+    $exitCode = $process.ExitCode
+    if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
+        echo "Original BuildTools uninstall failed with code $exitCode"
+        exit 1
     }
+    echo "Other versioned BuildTools uninstalled."
 }
 
+echo "Installing Visual Studio version ${env:VS_VERSION}."
 $process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
 Remove-Item -Path vs_installer.exe -Force
 $exitCode = $process.ExitCode
 if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
-    echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]."
+    echo "VS 2019 installer exited with code $exitCode, which should be one of [0, 3010]."
     curl.exe --retry 3 -kL $COLLECT_DOWNLOAD_LINK --output Collect.exe
     if ($LASTEXITCODE -ne 0) {
         echo "Download of the VS Collect tool failed."
         exit 1
     }
     Start-Process "${PWD}\Collect.exe" -NoNewWindow -Wait -PassThru
     New-Item -Path "C:\w\build-results" -ItemType "directory" -Force
-    Copy-Item -Path "C:\Users\circleci\AppData\Local\Temp\vslogs.zip" -Destination "C:\w\build-results\"
+    Copy-Item -Path "${env:TEMP}\vslogs.zip" -Destination "C:\w\build-results\"
     exit 1
 }
diff --git a/.circleci/verbatim-sources/build-parameters/pytorch-build-params.yml b/.circleci/verbatim-sources/build-parameters/pytorch-build-params.yml
@@ -89,6 +89,9 @@ pytorch_windows_params: &pytorch_windows_params
     python_version:
       type: string
       default: "3.8"
+    vs_version:
+      type: string
+      default: "16.8.6"
     vc_version:
       type: string
       default: "14.16"
@@ -106,6 +109,7 @@ pytorch_windows_params: &pytorch_windows_params
     SCCACHE_BUCKET: "ossci-compiler-cache"
     CUDA_VERSION: <<parameters.cuda_version>>
     PYTHON_VERSION: <<parameters.python_version>>
+    VS_VERSION: <<parameters.vs_version>>
     VC_VERSION: <<parameters.vc_version>>
     VC_YEAR: <<parameters.vc_year>>
     VC_PRODUCT: <<parameters.vc_product>>

diff --git a/.circleci/verbatim-sources/job-specs/job-specs-custom.yml b/.circleci/verbatim-sources/job-specs/job-specs-custom.yml
@@ -208,12 +208,11 @@
             set -ex
             source /Users/distiller/workspace/miniconda3/bin/activate
             pip install boto3
-            export PYTHONPATH="$PWD"
 
             # Using the same IAM user to write stats to our OSS bucket
             export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_SCCACHE_S3_BUCKET_V4}
             export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_SCCACHE_S3_BUCKET_V4}
-            python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test
+            python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
           when: always
       - store_test_results:
           path: test/test-reports

diff --git a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml b/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml
@@ -223,8 +223,7 @@ jobs:
           export JOB_BASE_NAME="$CIRCLE_JOB"
           export CIRCLE_WORKFLOW_ID="$CIRCLE_WORKFLOW_ID"
           cd workspace
-          export PYTHONPATH="\${PWD}"
-          python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test
+          python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
           EOL
           echo "(cat docker_commands.sh | docker exec -u jenkins -e LANG=C.UTF-8 -i "$id" bash) 2>&1" > command.sh
           unbuffer bash command.sh | ts
@@ -254,6 +253,9 @@ jobs:
       python_version:
         type: string
         default: "3.8"
+      vs_version:
+        type: string
+        default: "16.8.6"
       vc_version:
         type: string
         default: "14.16"
@@ -321,6 +323,9 @@ jobs:
       python_version:
         type: string
         default: "3.8"
+      vs_version:
+        type: string
+        default: "16.8.6"
       vc_version:
         type: string
         default: "14.16"
@@ -376,9 +381,8 @@ jobs:
             set -ex
             export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_WIN_BUILD_V1}
             export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_WIN_BUILD_V1}
-            export PYTHONPATH="$PWD"
             pip install typing_extensions boto3
-            python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test
+            python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
           when: always
       - store_test_results:
           path: test/test-reports

diff --git a/.circleci/verbatim-sources/workflows/workflows-scheduled-ci.yml b/.circleci/verbatim-sources/workflows/workflows-scheduled-ci.yml
@@ -85,6 +85,7 @@
           python_version: "3.8"
           use_cuda: "1"
           vc_product: BuildTools
+          vs_version: "16.8.6"
           vc_version: "14.28.29333"
           vc_year: "2019"
           filters:
@@ -103,6 +104,7 @@
           test_name: pytorch-windows-test1
           use_cuda: "1"
           vc_product: BuildTools
+          vs_version: "16.8.6"
           vc_version: "14.28.29333"
           vc_year: "2019"
           filters:
@@ -121,6 +123,7 @@
           test_name: pytorch-windows-test2
           use_cuda: "1"
           vc_product: BuildTools
+          vs_version: "16.8.6"
           vc_version: "14.28.29333"
           vc_year: "2019"
           filters:

diff --git a/.github/scripts/generate_ci_workflows.py b/.github/scripts/generate_ci_workflows.py
@@ -54,7 +54,11 @@ def PyTorchLinuxWorkflow(
     test_runner_type: str,
     on_pull_request: bool = False,
     enable_doc_jobs: bool = False,
+    enable_jit_legacy_test: YamlShellBool = "''",
     enable_multigpu_test: YamlShellBool = "''",
+    enable_nogpu_no_avx_test: YamlShellBool = "''",
+    enable_nogpu_no_avx2_test: YamlShellBool = "''",
+    enable_slow_test: YamlShellBool = "''",
     num_test_shards: int = 1,
     is_scheduled: Optional[str] = None,
 ) -> PyTorchWorkflow:
@@ -65,7 +69,11 @@ def PyTorchLinuxWorkflow(
         "on_pull_request": on_pull_request,
         "is_scheduled": is_scheduled,
         "enable_doc_jobs": enable_doc_jobs,
+        "enable_jit_legacy_test": enable_jit_legacy_test,
         "enable_multigpu_test": enable_multigpu_test,
+        "enable_nogpu_no_avx_test": enable_nogpu_no_avx_test,
+        "enable_nogpu_no_avx2_test": enable_nogpu_no_avx2_test,
+        "enable_slow_test": enable_slow_test,
         "num_test_shards": num_test_shards,
     }
 
@@ -163,7 +171,11 @@ def generate_workflow_file(
         build_environment="pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7",
         docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
         test_runner_type=LINUX_CUDA_TEST_RUNNER,
+        enable_jit_legacy_test=1,
         enable_multigpu_test=1,
+        enable_nogpu_no_avx_test=1,
+        enable_nogpu_no_avx2_test=1,
+        enable_slow_test=1,
         num_test_shards=2,
     ),
     PyTorchLinuxWorkflow(

diff --git a/.github/scripts/generate_pytorch_test_matrix.py b/.github/scripts/generate_pytorch_test_matrix.py
@@ -21,11 +21,21 @@ class Config(TypedDict):
 
 def main() -> None:
     TEST_RUNNER_TYPE = os.getenv('TEST_RUNNER_TYPE')
+    assert TEST_RUNNER_TYPE is not None
     NUM_TEST_SHARDS = int(os.getenv('NUM_TEST_SHARDS', '1'))
     MULTIGPU_RUNNER_TYPE = os.getenv('MULTIGPU_RUNNER_TYPE')
+    NOGPU_RUNNER_TYPE = os.getenv('NOGPU_RUNNER_TYPE')
     configs: Dict[str, Config] = {}
+    if os.getenv('ENABLE_JIT_LEGACY_TEST'):
+        configs['jit_legacy'] = {'num_shards': 1, 'runner': TEST_RUNNER_TYPE}
     if MULTIGPU_RUNNER_TYPE is not None and os.getenv('ENABLE_MULTIGPU_TEST'):
         configs['multigpu'] = {'num_shards': 1, 'runner': MULTIGPU_RUNNER_TYPE}
+    if NOGPU_RUNNER_TYPE is not None and os.getenv('ENABLE_NOGPU_NO_AVX_TEST'):
+        configs['nogpu_NO_AVX'] = {'num_shards': 1, 'runner': NOGPU_RUNNER_TYPE}
+    if NOGPU_RUNNER_TYPE is not None and os.getenv('ENABLE_NOGPU_NO_AVX2_TEST'):
+        configs['nogpu_NO_AVX2'] = {'num_shards': 1, 'runner': NOGPU_RUNNER_TYPE}
+    if os.getenv('ENABLE_SLOW_TEST'):
+        configs['slow'] = {'num_shards': 1, 'runner': TEST_RUNNER_TYPE}
     matrix = {
         'include': [
             {

diff --git a/.github/templates/bazel_ci_workflow.yml.j2 b/.github/templates/bazel_ci_workflow.yml.j2
@@ -81,11 +81,10 @@ name: Bazel Linux CI (!{{ build_environment }})
           CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
           CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
         run: |
-          export PYTHONPATH=$PWD
           COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
           export COMMIT_TIME
           pip3 install requests
-          python3 .circleci/scripts/upload_binary_size_to_scuba.py || exit 0
+          python3 -m tools.stats.upload_binary_size_to_scuba || exit 0
       - name: Test PyTorch
         run: |
           export SHARD_NUMBER=0
@@ -194,6 +193,5 @@ name: Bazel Linux CI (!{{ build_environment }})
           CIRCLE_TAG: ${{ steps.parse-ref.outputs.tag }}
           CIRCLE_WORKFLOW_ID: '${{ github.run_id }}_${{ github.run_number }}'
         run: |
-          export PYTHONPATH=$PWD
-          python tools/stats/print_test_stats.py --upload-to-s3 --compare-with-s3 test
+          python -m tools.stats.print_test_stats --upload-to-s3 --compare-with-s3 test
 {%- endblock %}