diff --git a/test/Utils/update-verify-tests/expansion.swift b/test/Utils/update-verify-tests/expansion.swift index db1f80a5678ad..bb29d80fc3706 100644 --- a/test/Utils/update-verify-tests/expansion.swift +++ b/test/Utils/update-verify-tests/expansion.swift @@ -39,6 +39,14 @@ // RUN: %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/nested.swift // RUN: %diff %t/nested.swift %t/nested.swift.expected +// RUN: not %target-swift-frontend-verify -I %t -plugin-path %swift-plugin-dir -typecheck %t/unparsed.swift 2>%t/output.txt -Rmacro-expansions +// RUN: not %update-verify-tests < %t/output.txt | %FileCheck --check-prefix CHECK-UNPARSED %s + +// RUN: not %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/escaped.swift 2>%t/output.txt -Rmacro-expansions +// RUN: %update-verify-tests < %t/output.txt +// RUN: %target-swift-frontend-verify -load-plugin-library %t/%target-library-name(UnstringifyMacroDefinition) -typecheck %t/escaped.swift -Rmacro-expansions +// RUN: %diff %t/escaped.swift %t/escaped.swift.expected + //--- single.swift @attached(peer, names: overloaded) macro unstringifyPeer(_ s: String) = @@ -129,9 +137,9 @@ func foo(_ x: Int) { } """) //expected-expansion@+5:14{{ - // expected-error@3 {{cannot find 'b' in scope; did you mean 'x'?}} // expected-note@1 2{{'x' declared here}} // expected-error@2 {{cannot find 'a' in scope; did you mean 'x'?}} + // expected-error@3 {{cannot find 'b' in scope; did you mean 'x'?}} //}} func foo() {} @@ -238,3 +246,62 @@ func bar(_ y: Int) { // }} func bar() {} +//--- unparsed.h +// CHECK-UNPARSED: no files updated: found diagnostics in unparsed files TMP_DIR/unparsed.h +void foo(int len, int *p) __attribute__((swift_attr("@_SwiftifyImport(.countedBy(pointer: .param(2), count: \"len\"))"))); + +//--- module.modulemap +module UnparsedClang { + header "unparsed.h" + export * +} + +//--- unparsed.swift +import UnparsedClang + +func bar() { + let a: CInt = 1 + var b: CInt = 13 + foo(a, &b) +} + +//--- escaped.swift +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +@unstringifyPeer(""" +func foo(_ x: Int) { + let a = "\\(x)" + let b = "\\(x)" +} +""") +// NB: DiagnosticVerifier interprets "\\(x)" as "\(x)" +// expected-expansion@+3:30{{ +// expected-remark@2{{macro content: |let a = "\\(x)"|}} +// }} +func foo() { let _ = "\(2)" } + +//--- escaped.swift.expected +@attached(peer, names: overloaded) +macro unstringifyPeer(_ s: String) = + #externalMacro(module: "UnstringifyMacroDefinition", type: "UnstringifyPeerMacro") + +// expected-note@+1 6{{in expansion of macro 'unstringifyPeer' on global function 'foo()' here}} +@unstringifyPeer(""" +func foo(_ x: Int) { + let a = "\\(x)" + let b = "\\(x)" +} +""") +// NB: DiagnosticVerifier interprets "\\(x)" as "\(x)" +// expected-expansion@+8:30{{ +// expected-remark@1{{macro content: |func foo(_ x: Int) {|}} +// expected-warning@2{{initialization of immutable value 'a' was never used; consider replacing with assignment to '_' or removing it}} +// expected-remark@2{{macro content: | let a = "\\(x)"|}} +// expected-warning@3{{initialization of immutable value 'b' was never used; consider replacing with assignment to '_' or removing it}} +// expected-remark@3{{macro content: | let b = "\\(x)"|}} +// expected-remark@4{{macro content: |}|}} +// }} +func foo() { let _ = "\(2)" } + diff --git a/utils/update_verify_tests/core.py b/utils/update_verify_tests/core.py index 7475c437ff824..fe636deb4e3cf 100644 --- a/utils/update_verify_tests/core.py +++ b/utils/update_verify_tests/core.py @@ -1,5 +1,6 @@ import sys import re +from codecs import encode, decode DEBUG = False @@ -118,7 +119,9 @@ def relative_target(self): def col(self): # expected-expansion requires column. Otherwise only retain column info if it's already there. - if self._col and (self.category == "expansion" or self.is_from_source_file): + if self._col and ( + self.category == "expansion" or self.is_from_source_file + ): return self._col return None @@ -170,7 +173,9 @@ def render(self): if self.category == "expansion": return base_s + "{{" else: - return base_s + "{{" + self.diag_content + "}}" + # python trivia: raw strings can't end with a backslash + escaped_diag_s = self.diag_content.replace("\\", "\\\\") + return base_s + "{{" + escaped_diag_s + "}}" class ExpansionDiagClose: @@ -243,9 +248,12 @@ def parse_diag(line, filename, prefix): count = int(count_s) if count_s else 1 line.content = matched_re.sub("{{DIAG}}", s) + unescaped_diag_s = decode( + encode(diag_s, "utf-8", "backslashreplace"), "unicode-escape" + ) return Diag( check_prefix, - diag_s, + unescaped_diag_s, category_s, target_line_n, is_absolute, @@ -260,6 +268,7 @@ def parse_diag(line, filename, prefix): def add_line(new_line, lines): + assert new_line.line_n > 0 lines.insert(new_line.line_n - 1, new_line) for i in range(new_line.line_n, len(lines)): line = lines[i] @@ -296,7 +305,11 @@ def infer_line_context(target, line_n): ) reverse = ( True - if [other for other in target.targeting_diags if other.relative_target() < 0] + if [ + other + for other in target.targeting_diags + if other.relative_target() < 0 + ] else False ) @@ -423,7 +436,7 @@ def fold_expansions(lines): if line.diag.category == "closing": line.diag.parent.closer = line else: - line.line_n = len(line.diag.parent.nested_lines) + line.line_n = len(line.diag.parent.nested_lines) + 1 add_line(line, line.diag.parent.nested_lines) @@ -446,7 +459,10 @@ def error_refers_to_diag(diag_error, diag, target_line_n): return ( target_line_n == diag.absolute_target() and diag_error.category == diag.category - and (diag.category == "expansion" or diag_error.content == diag.diag_content) + and ( + diag.category == "expansion" + or diag_error.content == diag.diag_content + ) ) @@ -468,7 +484,9 @@ def find_other_targeting(lines, orig_lines, is_nested, diag_error): return other_diags -def update_lines(diag_errors, lines, orig_lines, prefix, filename, nested_context): +def update_lines( + diag_errors, lines, orig_lines, prefix, filename, nested_context +): for diag_error in diag_errors: if not isinstance(diag_error, NotFoundDiag): continue @@ -511,10 +529,13 @@ def update_lines(diag_errors, lines, orig_lines, prefix, filename, nested_contex if isinstance(diag_error, NestedDiag): if not diag.closer: whitespace = ( - diag.whitespace_strings[0] if diag.whitespace_strings else " " + diag.whitespace_strings[0] + if diag.whitespace_strings + else " " ) diag.closer = Line( - get_indent(diag.line.content) + "//" + whitespace + "}}\n", None + get_indent(diag.line.content) + "//" + whitespace + "}}\n", + None, ) update_lines( [diag_error.nested], @@ -533,13 +554,17 @@ def update_test_file(filename, diag_errors, prefix, updated_test_files): else: updated_test_files.add(filename) with open(filename, "r") as f: - lines = [Line(line, i + 1) for i, line in enumerate(f.readlines() + [""])] + lines = [ + Line(line, i + 1) for i, line in enumerate(f.readlines() + [""]) + ] orig_lines = list(lines) expansion_context = [] for line in lines: + dprint(f"parsing line {line.render()}") diag = parse_diag(line, filename, prefix) if diag: + dprint(f" parsed diag {diag.render()}") line.diag = diag if expansion_context: diag.parent = expansion_context[-1] @@ -549,6 +574,8 @@ def update_test_file(filename, diag_errors, prefix, updated_test_files): expansion_context.append(diag) elif diag.category == "closing": expansion_context.pop() + else: + dprint(f" no diag") fold_expansions(lines) update_lines(diag_errors, lines, orig_lines, prefix, filename, None) @@ -559,7 +586,7 @@ def update_test_file(filename, diag_errors, prefix, updated_test_files): f.write(line.render()) -def update_test_files(errors, prefix): +def update_test_files(errors, prefix, unparsed_files): errors_by_file = {} for error in errors: filename = error.file @@ -568,6 +595,8 @@ def update_test_files(errors, prefix): errors_by_file[filename].append(error) updated_test_files = set() for filename, diag_errors in errors_by_file.items(): + if filename in unparsed_files: + continue try: update_test_file(filename, diag_errors, prefix, updated_test_files) except KnownException as e: @@ -576,7 +605,12 @@ def update_test_files(errors, prefix): None, ) updated_files = list(updated_test_files) - assert updated_files + assert updated_files or unparsed_files + if not updated_files: + return ( + f"no files updated: found diagnostics in unparsed files {', '.join(unparsed_files)}", + None, + ) return (None, updated_files) @@ -586,7 +620,9 @@ def update_test_files(errors, prefix): // expected-error@+1{{asdf}} ~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~ """ -diag_error_re = re.compile(r"(\S+):(\d+):(\d+): error: expected (\S+) not produced") +diag_error_re = re.compile( + r"(\S+):(\d+):(\d+): error: expected (\S+) not produced" +) """ @@ -607,7 +643,9 @@ def update_test_files(errors, prefix): ^~~~ cannot find 'bar' in scope """ -diag_error_re3 = re.compile(r"(\S+):(\d+):(\d+): error: incorrect message found") +diag_error_re3 = re.compile( + r"(\S+):(\d+):(\d+): error: incorrect message found" +) """ @@ -617,7 +655,9 @@ def update_test_files(errors, prefix): ^~~~~~~ error """ -diag_error_re4 = re.compile(r"(\S+):(\d+):(\d+): error: expected (\S+), not (\S+)") +diag_error_re4 = re.compile( + r"(\S+):(\d+):(\d+): error: expected (\S+), not (\S+)" +) """ ex: @@ -625,7 +665,19 @@ def update_test_files(errors, prefix): func foo() {} ^ """ -diag_expansion_note_re = re.compile(r"(\S+):(\d+):(\d+): note: in expansion from here") +diag_expansion_note_re = re.compile( + r"(\S+):(\d+):(\d+): note: in expansion from here" +) + +""" +ex: +test.h:8:52: note: file 'test.h' is not parsed for 'expected' statements. Use '-verify-additional-file test.h' to enable, or '-verify-ignore-unrelated' to ignore diagnostics in this file +void foo(int len, int * __counted_by(len) p); + ^ +""" +diag_not_parsed_note_re = re.compile( + r"(\S+):(\d+):(\d+): note: file '(\S+)' is not parsed for 'expected' statements" +) class NotFoundDiag: @@ -678,19 +730,35 @@ def check_expectations(tool_output, prefix): Called by the stand-alone update-verify-tests.py as well as litplugin.py. """ top_level = [] + unparsed_files = set() try: i = 0 while i < len(tool_output): line = tool_output[i].strip() + extra_lines = [] curr = [] + dprint(f"line: {line.strip()}") if not "error:" in line: - pass + if "note:" in line: + if m := diag_not_parsed_note_re.match(line.strip()): + dprint(f"unparsed file: {m.group(4)}") + unparsed_files.add(m.group(4)) + extra_lines = tool_output[i + 1 : i + 3] + dprint(f"extra lines: {extra_lines}") + else: + raise KnownException( + f"unhandled note found (line {i+1}): '{line.strip()}'" + ) + else: + dprint(f"ignored line: {line.strip()}") elif m := diag_error_re.match(line): + dprint(f"diag not found: {line.strip()}") + extra_lines = tool_output[i + 1 : i + 3] + dprint(f"extra lines: {extra_lines}") diag = parse_diag( - Line(tool_output[i + 1], int(m.group(2))), m.group(1), prefix + Line(extra_lines[0], int(m.group(2))), m.group(1), prefix ) - i += 2 curr.append( NotFoundDiag( m.group(1), @@ -702,6 +770,9 @@ def check_expectations(tool_output, prefix): ) ) elif m := diag_error_re2.match(line): + dprint(f"unexpected diag: {line.strip()}") + extra_lines = tool_output[i + 1 : i + 3] + dprint(f"extra lines: {extra_lines}") curr.append( ExtraDiag( m.group(1), @@ -712,14 +783,16 @@ def check_expectations(tool_output, prefix): prefix, ) ) - i += 2 # Create two mirroring mismatches when the compiler reports that the category or diagnostic is incorrect. # This makes it easier to handle cases where the same diagnostic is mentioned both in an incorrect message/category # diagnostic, as well as in an error not produced diagnostic. This can happen for things like 'expected-error 2{{foo}}' # if only one diagnostic is emitted on that line, and the content of that diagnostic is actually 'bar'. elif m := diag_error_re3.match(line): + dprint(f"wrong diag message: {line.strip()}") + extra_lines = tool_output[i + 1 : i + 4] + dprint(f"extra lines: {extra_lines}") diag = parse_diag( - Line(tool_output[i + 1], int(m.group(2))), m.group(1), prefix + Line(extra_lines[0], int(m.group(2))), m.group(1), prefix ) curr.append( NotFoundDiag( @@ -737,17 +810,19 @@ def check_expectations(tool_output, prefix): diag.absolute_target(), int(m.group(3)), diag.category, - tool_output[i + 3].strip(), + extra_lines[2].strip(), diag.prefix, ) ) - i += 3 elif m := diag_error_re4.match(line): + dprint(f"wrong diag kind: {line.strip()}") + extra_lines = tool_output[i + 1 : i + 4] + dprint(f"extra lines: {extra_lines}") diag = parse_diag( - Line(tool_output[i + 1], int(m.group(2))), m.group(1), prefix + Line(extra_lines[0], int(m.group(2))), m.group(1), prefix ) assert diag.category == m.group(4) - assert tool_output[i + 3].strip() == m.group(5) + assert extra_lines[2].strip() == m.group(5) curr.append( NotFoundDiag( m.group(1), @@ -768,27 +843,30 @@ def check_expectations(tool_output, prefix): diag.prefix, ) ) - i += 3 else: - dprint("no match") - dprint(line.strip()) - i += 1 + dprint(f"no match: {line.strip()}") + i += 1 + len(extra_lines) while ( curr and i < len(tool_output) and (m := diag_expansion_note_re.match(tool_output[i].strip())) ): + nested_note_lines = tool_output[i : i + 3] + dprint(f"nested note lines: {nested_note_lines}") curr = [ NestedDiag(m.group(1), int(m.group(2)), int(m.group(3)), e) for e in curr ] - i += 3 + i += len(nested_note_lines) top_level.extend(curr) except KnownException as e: - return (f"Error in update-verify-tests while parsing tool output: {e}", None) + return ( + f"Error in update-verify-tests while parsing tool output: {e}", + None, + ) if top_level: - return update_test_files(top_level, prefix) + return update_test_files(top_level, prefix, unparsed_files) else: return ("no mismatching diagnostics found", None)