Fix failing test due to a bug in NumPy when using OpenBLAS (#67679)

Summary:
Pull Request resolved: #67679

implementations

Fixes #67675

cc mruberry

Test Plan: Imported from OSS

Reviewed By: anjali411

Differential Revision: D32368698

Pulled By: mruberry

fbshipit-source-id: 3ea6ebc43c061af2f376cdf5da06884859bbbf53
Signed-off-by: Eli Uriegas <eliuriegas@fb.com>
ghstack-source-id: 856e69d7e57f4e8cd8c794feda9487f006c7dfde

.github/scripts/generate_ci_workflows.py

@@ -157,6 +157,7 @@ class CIWorkflow:
     enable_doc_jobs: bool = False
     exclude_test: bool = False
     build_generates_artifacts: bool = True
+    build_with_debug: bool = False
     is_scheduled: str = ''
     num_test_shards: int = 1
     only_run_smoke_tests_on_pull_request: bool = False
@@ -221,6 +222,8 @@ def assert_valid(self) -> None:
         if self.is_scheduled:
             assert LABEL_CIFLOW_DEFAULT not in self.ciflow_config.labels
             assert LABEL_CIFLOW_SCHEDULED in self.ciflow_config.labels
+        if self.build_with_debug:
+            assert self.build_environment.endswith("-debug")
 
     def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
         output_file_path = GITHUB_DIR / f"workflows/generated-{self.build_environment}.yml"
@@ -451,10 +454,11 @@ def generate_workflow_file(self, workflow_template: jinja2.Template) -> None:
     ),
     CIWorkflow(
         arch="linux",
-        build_environment="periodic-linux-xenial-cuda11.1-py3.6-gcc7",
+        build_environment="periodic-linux-xenial-cuda11.1-py3.6-gcc7-debug",
         docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
         test_runner_type=LINUX_CUDA_TEST_RUNNER,
         num_test_shards=2,
+        build_with_debug=True,
         is_scheduled="45 0,4,8,12,16,20 * * *",
         ciflow_config=CIFlowConfig(
             labels={LABEL_CIFLOW_SCHEDULED, LABEL_CIFLOW_LINUX, LABEL_CIFLOW_CUDA}

.github/templates/linux_ci_workflow.yml.j2

@@ -39,6 +39,9 @@ env:
   AWS_DEFAULT_REGION: us-east-1
   PR_NUMBER: ${{ github.event.pull_request.number }}
   SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
+{%- if build_with_debug %}
+  DEBUG: 1
+{%- endif %}
 !{{ common.concurrency(build_environment) }}
 
 jobs:

.github/templates/windows_ci_workflow.yml.j2

@@ -52,6 +52,9 @@ env:
   AWS_DEFAULT_REGION: us-east-1
   PR_NUMBER: ${{ github.event.pull_request.number }}
   SHA1: ${{ github.event.pull_request.head.sha || github.sha }}
+{%- if build_with_debug %}
+  DEBUG: 1
+{%- endif %}
 {%- if cuda_version != "cpu" %}
   TORCH_CUDA_ARCH_LIST: "7.0"
 {%- endif %}

test/test_linalg.py

@@ -1638,7 +1638,12 @@ def run_test_case(input, p):
         a = torch.eye(3, dtype=dtype, device=device)
         a[-1, -1] = 0  # make 'a' singular
         for p in norm_types:
-            run_test_case(a, p)
+            try:
+                run_test_case(a, p)
+            except np.linalg.LinAlgError:
+                # Numpy may fail to converge for some BLAS backends (although this is very rare)
+                # See the discussion in https://github.com/pytorch/pytorch/issues/67675
+                pass
 
         # test for 0x0 matrices. NumPy doesn't work for such input, we return 0
         input_sizes = [(0, 0), (2, 5, 0, 0)]