Merge branch 'master' into blacktoblock

.circleci/cimodel/data/binary_build_data.py

@@ -50,7 +50,8 @@ def get_processor_arch_name(cuda_version):
             "3.7",
         ],
     )),
-    windows=(dimensions.CUDA_VERSIONS, OrderedDict(
+    # Skip CUDA-9.2 builds on Windows
+    windows=([v for v in dimensions.CUDA_VERSIONS if v != '92'], OrderedDict(
         wheel=dimensions.STANDARD_PYTHON_VERSIONS,
         conda=dimensions.STANDARD_PYTHON_VERSIONS,
         libtorch=[

.circleci/cimodel/data/pytorch_build_data.py

@@ -60,9 +60,6 @@
         ("gcc", [
             ("9", [XImportant("3.8")]),
         ]),
-        ("conda", [
-            ("", [XImportant("3.7")]),
-        ]),
     ]),
 ]
 

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -2,13 +2,12 @@
 from dataclasses import dataclass, field
 from typing import List, Optional
 
-from cimodel.data.pytorch_build_data import TopLevelNode, CONFIG_TREE_DATA
 import cimodel.data.dimensions as dimensions
 import cimodel.lib.conf_tree as conf_tree
 import cimodel.lib.miniutils as miniutils
+from cimodel.data.pytorch_build_data import CONFIG_TREE_DATA, TopLevelNode
 from cimodel.data.simple.util.branch_filters import gen_filter_dict
-from cimodel.data.simple.util.docker_constants import gen_docker_image_path
-
+from cimodel.data.simple.util.docker_constants import gen_docker_image
 
 
 @dataclass
@@ -27,14 +26,14 @@ class Conf:
     restrict_phases: Optional[List[str]] = None
     gpu_resource: Optional[str] = None
     dependent_tests: List = field(default_factory=list)
-    parent_build: Optional['Conf'] = None
+    parent_build: Optional["Conf"] = None
     is_libtorch: bool = False
     is_important: bool = False
     parallel_backend: Optional[str] = None
 
     @staticmethod
     def is_test_phase(phase):
-        return phase in ["test", "test1", "test2"]
+        return "test" in phase
 
     # TODO: Eliminate the special casing for docker paths
     # In the short term, we *will* need to support special casing as docker images are merged for caffe2 and pytorch
@@ -64,18 +63,26 @@ def get_parms(self, for_docker):
         return result
 
     def gen_docker_image_path(self):
-
         parms_source = self.parent_build or self
         base_build_env_name = "-".join(parms_source.get_parms(True))
+        image_name, _ = gen_docker_image(base_build_env_name)
+        return miniutils.quote(image_name)
 
-        image_path = gen_docker_image_path(base_build_env_name)
-        return miniutils.quote(image_path)
+    def gen_docker_image_requires(self):
+        parms_source = self.parent_build or self
+        base_build_env_name = "-".join(parms_source.get_parms(True))
+        _, requires = gen_docker_image(base_build_env_name)
+        return miniutils.quote(requires)
 
     def get_build_job_name_pieces(self, build_or_test):
         return self.get_parms(False) + [build_or_test]
 
     def gen_build_name(self, build_or_test):
-        return ("_".join(map(str, self.get_build_job_name_pieces(build_or_test)))).replace(".", "_").replace("-", "_")
+        return (
+            ("_".join(map(str, self.get_build_job_name_pieces(build_or_test))))
+            .replace(".", "_")
+            .replace("-", "_")
+        )
 
     def get_dependents(self):
         return self.dependent_tests or []
@@ -116,12 +123,13 @@ def gen_workflow_job(self, phase):
             job_name = "pytorch_linux_test"
         else:
             job_name = "pytorch_linux_build"
+            job_def["requires"] = [self.gen_docker_image_requires()]
 
         if not self.is_important:
             job_def["filters"] = gen_filter_dict()
         job_def.update(self.gen_workflow_params(phase))
 
-        return {job_name : job_def}
+        return {job_name: job_def}
 
 
 # TODO This is a hack to special case some configs just for the workflow list
@@ -131,20 +139,26 @@ def __init__(self, name, parent_build=None):
         self.parent_build = parent_build
 
     def gen_workflow_job(self, phase):
-        return {self.gen_build_name(phase): {"requires": [self.parent_build.gen_build_name("build")]}}
+        return {
+            self.gen_build_name(phase): {
+                "requires": [self.parent_build.gen_build_name("build")]
+            }
+        }
 
     def gen_build_name(self, _):
         return self.name
 
 class DocPushConf(object):
-    def __init__(self, name, parent_build=None):
+    def __init__(self, name, parent_build=None, branch="master"):
         self.name = name
         self.parent_build = parent_build
+        self.branch = branch
 
     def gen_workflow_job(self, phase):
         return {
             "pytorch_doc_push": {
                 "name": self.name,
+                "branch": self.branch,
                 "requires": [self.parent_build],
                 "context": "org-member",
                 "filters": gen_filter_dict(branches_list=["nightly"])
@@ -183,12 +197,40 @@ def gen_dependent_configs(xenial_parent_config):
 def gen_docs_configs(xenial_parent_config):
     configs = []
 
-    for x in ["pytorch_python_doc_build", "pytorch_cpp_doc_build"]:
-        conf = HiddenConf(x, parent_build=xenial_parent_config)
-        configs.append(conf)
-        configs.append(DocPushConf(x.replace("build", "push"), x))
+    configs.append(
+        HiddenConf(
+            "pytorch_python_doc_build",
+            parent_build=xenial_parent_config
+        )
+    )
+    configs.append(
+        DocPushConf(
+            "pytorch_python_doc_push",
+            parent_build="pytorch_python_doc_build",
+            branch="site",
+        )
+    )
 
-    configs.append(HiddenConf("pytorch_doc_test", parent_build=xenial_parent_config))
+    configs.append(
+        HiddenConf(
+            "pytorch_cpp_doc_build",
+            parent_build=xenial_parent_config
+        )
+    )
+    configs.append(
+        DocPushConf(
+            "pytorch_cpp_doc_push",
+            parent_build="pytorch_cpp_doc_build",
+            branch="master",
+        )
+    )
+
+    configs.append(
+        HiddenConf(
+            "pytorch_doc_test",
+            parent_build=xenial_parent_config
+        )
+    )
     return configs
 
 
@@ -235,7 +277,7 @@ def instantiate_configs():
 
         elif compiler_name == "rocm":
             rocm_version = fc.find_prop("compiler_version")
-            restrict_phases = ["build", "test1", "test2"]
+            restrict_phases = ["build", "test1", "test2", "caffe2_test"]
 
         elif compiler_name == "android":
             android_ndk_version = fc.find_prop("compiler_version")
@@ -251,7 +293,7 @@ def instantiate_configs():
             parms_list.append(gcc_version)
 
             # TODO: This is a nasty special case
-            if gcc_version == 'clang5' and not is_xla:
+            if gcc_version == "clang5" and not is_xla:
                 parms_list.append("asan")
                 python_version = fc.find_prop("pyver")
                 parms_list[0] = fc.find_prop("abbreviated_pyver")
@@ -290,20 +332,25 @@ def instantiate_configs():
 
         # run docs builds on "pytorch-linux-xenial-py3.6-gcc5.4". Docs builds
         # should run on a CPU-only build that runs on all PRs.
-        if distro_name == 'xenial' and fc.find_prop("pyver") == '3.6' \
-                and cuda_version is None \
-                and parallel_backend is None \
-                and compiler_name == 'gcc' \
-                and fc.find_prop('compiler_version') == '5.4':
+        if (
+            distro_name == "xenial"
+            and fc.find_prop("pyver") == "3.6"
+            and cuda_version is None
+            and parallel_backend is None
+            and compiler_name == "gcc"
+            and fc.find_prop("compiler_version") == "5.4"
+        ):
             c.dependent_tests = gen_docs_configs(c)
 
         if cuda_version == "10.1" and python_version == "3.6" and not is_libtorch:
             c.dependent_tests = gen_dependent_configs(c)
 
-        if (compiler_name == "gcc"
-                and compiler_version == "5.4"
-                and not is_libtorch
-                and parallel_backend is None):
+        if (
+            compiler_name == "gcc"
+            and compiler_version == "5.4"
+            and not is_libtorch
+            and parallel_backend is None
+        ):
             bc_breaking_check = Conf(
                 "backward-compatibility-check",
                 [],

.circleci/cimodel/data/simple/android_definitions.py

@@ -1,5 +1,7 @@
 import cimodel.data.simple.util.branch_filters as branch_filters
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_NDK
+from cimodel.data.simple.util.docker_constants import (
+    DOCKER_IMAGE_NDK, DOCKER_REQUIREMENT_NDK
+)
 
 
 class AndroidJob:
@@ -34,6 +36,7 @@ def gen_tree(self):
             "name": full_job_name,
             "build_environment": "\"{}\"".format(build_env_name),
             "docker_image": "\"{}\"".format(DOCKER_IMAGE_NDK),
+            "requires": [DOCKER_REQUIREMENT_NDK]
         }
 
         if self.is_master_only:

.circleci/cimodel/data/simple/bazel_definitions.py

@@ -1,4 +1,7 @@
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_GCC7
+from cimodel.data.simple.util.docker_constants import (
+    DOCKER_IMAGE_GCC7,
+    DOCKER_REQUIREMENT_GCC7
+)
 
 
 def gen_job_name(phase):
@@ -38,7 +41,10 @@ def gen_tree(self):
         full_job_name = gen_job_name(self.phase)
         build_env_name = "-".join(build_env_parts)
 
-        extra_requires = [gen_job_name("build")] if self.phase == "test" else []
+        extra_requires = (
+            [gen_job_name("build")] if self.phase == "test" else
+            [DOCKER_REQUIREMENT_GCC7]
+        )
 
         props_dict = {
             "build_environment": build_env_name,

.circleci/cimodel/data/simple/docker_definitions.py

@@ -28,7 +28,6 @@
     "pytorch-linux-xenial-py3.6-gcc7",
     "pytorch-linux-xenial-rocm3.3-py3.6",
     "pytorch-linux-xenial-rocm3.5.1-py3.6",
-    "pytorch-linux-bionic-py3.7-conda",
 ]
 
 

.circleci/cimodel/data/simple/mobile_definitions.py

@@ -4,12 +4,23 @@
 
 import cimodel.lib.miniutils as miniutils
 import cimodel.data.simple.util.branch_filters
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_ASAN, DOCKER_IMAGE_NDK
+from cimodel.data.simple.util.docker_constants import (
+    DOCKER_IMAGE_ASAN,
+    DOCKER_REQUIREMENT_ASAN,
+    DOCKER_IMAGE_NDK,
+    DOCKER_REQUIREMENT_NDK
+)
 
 
 class MobileJob:
-    def __init__(self, docker_image, variant_parts, is_master_only=False):
+    def __init__(
+            self,
+            docker_image,
+            docker_requires,
+            variant_parts,
+            is_master_only=False):
         self.docker_image = docker_image
+        self.docker_requires = docker_requires
         self.variant_parts = variant_parts
         self.is_master_only = is_master_only
 
@@ -30,6 +41,7 @@ def gen_tree(self):
             "build_environment": build_env_name,
             "build_only": miniutils.quote(str(int(True))),
             "docker_image": self.docker_image,
+            "requires": self.docker_requires,
             "name": full_job_name,
         }
 
@@ -40,14 +52,32 @@ def gen_tree(self):
 
 
 WORKFLOW_DATA = [
-    MobileJob(DOCKER_IMAGE_ASAN, ["build"]),
-    MobileJob(DOCKER_IMAGE_ASAN, ["custom", "build", "static"]),
+    MobileJob(
+        DOCKER_IMAGE_ASAN,
+        [DOCKER_REQUIREMENT_ASAN],
+        ["build"]
+    ),
+    MobileJob(
+        DOCKER_IMAGE_ASAN,
+        [DOCKER_REQUIREMENT_ASAN],
+        ["custom", "build", "static"]
+    ),
 
     # Use LLVM-DEV toolchain in android-ndk-r19c docker image
-    MobileJob(DOCKER_IMAGE_NDK, ["custom", "build", "dynamic"]),
+    MobileJob(
+        DOCKER_IMAGE_NDK,
+        [DOCKER_REQUIREMENT_NDK],
+        ["custom", "build", "dynamic"]
+    ),
 
     # Use LLVM-DEV toolchain in android-ndk-r19c docker image
-    MobileJob(DOCKER_IMAGE_NDK, ["code", "analysis"]),
+    # Most of this CI is already covered by "mobile-custom-build-dynamic" job
+    MobileJob(
+        DOCKER_IMAGE_NDK,
+        [DOCKER_REQUIREMENT_NDK],
+        ["code", "analysis"],
+        True
+    ),
 ]
 
 

.circleci/cimodel/data/simple/nightly_android.py

@@ -1,4 +1,7 @@
-from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_NDK
+from cimodel.data.simple.util.docker_constants import (
+    DOCKER_IMAGE_NDK,
+    DOCKER_REQUIREMENT_NDK
+)
 
 
 class AndroidNightlyJob:
@@ -48,12 +51,13 @@ def gen_tree(self):
 
         return [{self.template_name: props_dict}]
 
+BASE_REQUIRES = [DOCKER_REQUIREMENT_NDK]
 
 WORKFLOW_DATA = [
-    AndroidNightlyJob(["x86_32"], "pytorch_linux_build"),
-    AndroidNightlyJob(["x86_64"], "pytorch_linux_build"),
-    AndroidNightlyJob(["arm", "v7a"], "pytorch_linux_build"),
-    AndroidNightlyJob(["arm", "v8a"], "pytorch_linux_build"),
+    AndroidNightlyJob(["x86_32"], "pytorch_linux_build", requires=BASE_REQUIRES),
+    AndroidNightlyJob(["x86_64"], "pytorch_linux_build", requires=BASE_REQUIRES),
+    AndroidNightlyJob(["arm", "v7a"], "pytorch_linux_build", requires=BASE_REQUIRES),
+    AndroidNightlyJob(["arm", "v8a"], "pytorch_linux_build", requires=BASE_REQUIRES),
     AndroidNightlyJob(["android_gradle"], "pytorch_android_gradle_build",
                       with_docker=False,
                       requires=[