[UpdateTestChecks] Don't fail silently when conflicting CHECK lines means no checks are generated for some functions (#159321)

There is a warning that triggers if you (for instance) run
`update_llc_test_checks.py` on an input where _all_ functions have
conflicting check lines and so no checks are generated. However, there
are no warnings emitted at all for the case where some functions have
non-conflicting check lines but others don't. This is a source of
frustration because running update_llc_test_checks can result in all
check lines being removed for certain functions when such a conflict
exists with no warning, meaning we have to be extra vigilant inspecting
the diff. I've also personally wasted time tracking down the source of
the dropped lines assuming that update_test_checks would emit a warning
in such cases.

This change adds logic to emit warnings on a function-by-function basis
for any RUN that has conflicting prefixes meaning no output is
generated. This subsumes the previous warning for when _all_ functions
conflict.

Author: asb

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll

@@ -0,0 +1,16 @@
+; RUN: sed 's/RETVAL/1/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/RETVAL/2/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/RETVAL/3/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKB %s
+; RUN: sed 's/RETVAL/4/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKB %s
+
+define i32 @foo() {
+  ret i32 RETVAL
+}
+
+define i32 @bar() {
+  ret i32 100
+}

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/conflicting-prefixes.ll.expected

@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no-generate-body-for-unused-prefixes
+; RUN: sed 's/RETVAL/1/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/RETVAL/2/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/RETVAL/3/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKB %s
+; RUN: sed 's/RETVAL/4/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKB %s
+
+define i32 @foo() {
+  ret i32 RETVAL
+}
+
+define i32 @bar() {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 100
+; CHECK-NEXT:    ret
+  ret i32 100
+}

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll

@@ -0,0 +1,14 @@
+; RUN: sed 's/FN/foo/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/FN/foo/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKB %s
+; RUN: sed 's/FN/bar/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKC %s
+
+define i32 @FN() {
+  ret i32 1
+}
+
+define i32 @common() {
+  ret i32 100
+}

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/differing-set-of-functions.ll.expected

@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --include-generated-funcs
+; RUN: sed 's/FN/foo/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKA %s
+; RUN: sed 's/FN/foo/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKB %s
+; RUN: sed 's/FN/bar/g' %s | llc -mtriple=riscv32 \
+; RUN:   | FileCheck -check-prefixes=CHECK,CHECKC %s
+
+define i32 @FN() {
+  ret i32 1
+}
+
+define i32 @common() {
+  ret i32 100
+}
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+;
+; CHECK-LABEL: common:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 100
+; CHECK-NEXT:    ret
+;
+; CHECKA-LABEL: foo:
+; CHECKA:       # %bb.0:
+; CHECKA-NEXT:    li a0, 1
+; CHECKA-NEXT:    ret
+;
+; CHECKA-LABEL: common:
+; CHECKA:       # %bb.0:
+; CHECKA-NEXT:    li a0, 100
+; CHECKA-NEXT:    ret
+;
+; CHECKB-LABEL: foo:
+; CHECKB:       # %bb.0:
+; CHECKB-NEXT:    li a0, 1
+; CHECKB-NEXT:    ret
+;
+; CHECKB-LABEL: common:
+; CHECKB:       # %bb.0:
+; CHECKB-NEXT:    li a0, 100
+; CHECKB-NEXT:    ret
+;
+; CHECKC-LABEL: bar:
+; CHECKC:       # %bb.0:
+; CHECKC-NEXT:    li a0, 1
+; CHECKC-NEXT:    ret
+;
+; CHECKC-LABEL: common:
+; CHECKC:       # %bb.0:
+; CHECKC-NEXT:    li a0, 100
+; CHECKC-NEXT:    ret

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/conflicting-prefixes.test

@@ -0,0 +1,7 @@
+# REQUIRES: riscv-registered-target
+
+# RUN: cp -f %S/Inputs/conflicting-prefixes.ll %t.ll
+# RUN: %update_llc_test_checks --no-generate-body-for-unused-prefixes %t.ll 2>&1 | FileCheck %s
+# RUN: diff -u %S/Inputs/conflicting-prefixes.ll.expected %t.ll
+
+# CHECK: WARNING: For function 'foo', the following RUN lines will not generate checks due to conflicting output: RUN #1 (prefixes: CHECK, CHECKA), RUN #2 (prefixes: CHECK, CHECKA), RUN #3 (prefixes: CHECK, CHECKB), RUN #4 (prefixes: CHECK, CHECKB):

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/differing-set-of-functions.test

@@ -0,0 +1,11 @@
+# REQUIRES: riscv-registered-target
+
+# RUN: cp -f %S/Inputs/differing-set-of-functions.ll %t.ll
+# RUN: %update_llc_test_checks --include-generated-funcs %t.ll 2>&1 | FileCheck --allow-empty %s
+# RUN: diff -u %S/Inputs/differing-set-of-functions.ll.expected %t.ll
+
+# We shouldn't print the warning for clashing CHECK prefixes in the case that
+# we're trying to handle a function that is only present for some RUN lines.
+# Better warning behaviour than this might be possible.
+
+# CHECK-NOT: WARNING

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/prefix-never-matches.test

@@ -4,5 +4,5 @@
 # RUN: %update_llc_test_checks --no-generate-body-for-unused-prefixes %t.ll 2>&1 | FileCheck %s
 # RUN: FileCheck --input-file=%t.ll %s --check-prefix=OUTPUT
 
-# CHECK: WARNING: Prefix A had conflicting output
+# CHECK: WARNING: For function 'fold_v2i64', the following RUN lines will not generate checks due to conflicting output
 # OUTPUT-NOT: A:

llvm/utils/UpdateTestChecks/common.py

@@ -882,6 +882,7 @@ def __str__(self):
 
 class FunctionTestBuilder:
     def __init__(self, run_list, flags, scrubber_args, path, ginfo):
+        self._run_list = run_list
         self._verbose = flags.verbose
         self._record_args = flags.function_signature
         self._check_attributes = flags.check_attributes
@@ -917,15 +918,53 @@ def __init__(self, run_list, flags, scrubber_args, path, ginfo):
                 self._func_order.update({prefix: []})
                 self._global_var_dict.update({prefix: dict()})
 
+    # Return true if there is conflicting output for different runs for the
+    # given prefix and function name.
+    def has_conflicting_output(self, prefix, func):
+        # There was conflicting output if the func_dict is None for this
+        # prefix and function.
+        return self._func_dict[prefix].get(func) is None
+
     def finish_and_get_func_dict(self):
-        for prefix in self.get_failed_prefixes():
-            warn(
-                "Prefix %s had conflicting output from different RUN lines for all functions in test %s"
-                % (
-                    prefix,
-                    self._path,
+        all_funcs = set()
+        for prefix in self._func_dict:
+            all_funcs.update(self._func_dict[prefix].keys())
+
+        warnings_to_print = collections.defaultdict(list)
+        for func in sorted(list(all_funcs)):
+            for i, run_info in enumerate(self._run_list):
+                prefixes = run_info[0]
+                if not prefixes:
+                    continue
+
+                # Check if this RUN line produces this function at all. If
+                # not, we can skip analysing this function for this RUN.
+                run_contains_func = all(
+                    func in self._func_dict.get(p, {}) for p in prefixes
                 )
+                if not run_contains_func:
+                    continue
+
+                # Check if this RUN line can print any checks for this
+                # function. It can't if all of its prefixes have conflicting
+                # (None) output.
+                cannot_print_for_this_run = all(
+                    self.has_conflicting_output(p, func) for p in prefixes
+                )
+                if cannot_print_for_this_run:
+                    warnings_to_print[func].append((i, prefixes))
+
+        for func, warning_info in warnings_to_print.items():
+            conflict_strs = []
+            for run_index, prefixes in warning_info:
+                conflict_strs.append(
+                    f"RUN #{run_index + 1} (prefixes: {', '.join(prefixes)})"
+                )
+            warn(
+                f"For function '{func}', the following RUN lines will not generate checks due to conflicting output: {', '.join(conflict_strs)}",
+                test_file=self._path,
             )
+
         return self._func_dict
 
     def func_order(self):
@@ -1078,20 +1117,6 @@ def processed_prefixes(self, prefixes):
         """
         self._processed_prefixes.update(prefixes)
 
-    def get_failed_prefixes(self):
-        # This returns the list of those prefixes that failed to match any function,
-        # because there were conflicting bodies produced by different RUN lines, in
-        # all instances of the prefix.
-        for prefix in self._func_dict:
-            if self._func_dict[prefix] and (
-                not [
-                    fct
-                    for fct in self._func_dict[prefix]
-                    if self._func_dict[prefix][fct] is not None
-                ]
-            ):
-                yield prefix
-
 
 ##### Generator of LLVM IR CHECK lines