[utils] add update-verify-tests.py

test/Utils/update-verify-tests/diag-at-eof.swift

@@ -0,0 +1,15 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+
+//--- test.swift.expected
+// expected-note@+1{{to match this opening '{'}}
+func foo() {
+
+// expected-error@+1{{expected '}' at end of brace statement}}

test/Utils/update-verify-tests/duplicate-diag.swift

@@ -0,0 +1,27 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+  // expected-error@+1{{cannot find 'a' in scope}}
+  a = 2; a = 2;
+  b = 2; b = 2;
+
+  // expected-error@+1 3{{cannot find 'c' in scope}}
+  c = 2; c = 2;
+  // expected-error 3{{asdf}}
+}
+//--- test.swift.expected
+func foo() {
+  // expected-error@+1 2{{cannot find 'a' in scope}}
+  a = 2; a = 2;
+  // expected-error@+1 2{{cannot find 'b' in scope}}
+  b = 2; b = 2;
+
+  // expected-error@+1 2{{cannot find 'c' in scope}}
+  c = 2; c = 2;
+}

test/Utils/update-verify-tests/infer-indentation.swift

@@ -0,0 +1,28 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+         //     expected-error@+1    2{{cannot find 'a' in scope}}
+    a = 2; a = 2; b = 2; b = 2; c = 2;
+         //     expected-error@+1    2{{asdf}}
+    d = 2;
+    e = 2; f = 2;                 //     expected-error    2{{cannot find 'e' in scope}}
+}
+
+//--- test.swift.expected
+func foo() {
+         //     expected-error@+3    {{cannot find 'c' in scope}}
+         //     expected-error@+2    2{{cannot find 'b' in scope}}
+         //     expected-error@+1    2{{cannot find 'a' in scope}}
+    a = 2; a = 2; b = 2; b = 2; c = 2;
+         //     expected-error@+1    {{cannot find 'd' in scope}}
+    d = 2;
+    //     expected-error@+1    {{cannot find 'f' in scope}}
+    e = 2; f = 2;                 //     expected-error    {{cannot find 'e' in scope}}
+}
+

test/Utils/update-verify-tests/leave-existing-diags.swift

@@ -0,0 +1,32 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+    a = 2;
+    // expected-error@-1{{cannot find 'a' in scope}}
+    b = 2;// expected-error{{cannot find 'b' in scope}}
+    c = 2;
+    // expected-error@5{{cannot find 'c' in scope}}
+    d = 2; // expected-error{{'d' in scope}}
+
+    e = 2; // error to trigger mismatch
+}
+
+//--- test.swift.expected
+func foo() {
+    a = 2;
+    // expected-error@-1{{cannot find 'a' in scope}}
+    b = 2;// expected-error{{cannot find 'b' in scope}}
+    c = 2;
+    // expected-error@5{{cannot find 'c' in scope}}
+    d = 2; // expected-error{{'d' in scope}}
+
+    // expected-error@+1{{cannot find 'e' in scope}}
+    e = 2; // error to trigger mismatch
+}
+

test/Utils/update-verify-tests/multiple-errors.swift

@@ -0,0 +1,24 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+  a = 2
+  b = 2
+
+  c = 2
+}
+//--- test.swift.expected
+func foo() {
+  // expected-error@+1{{cannot find 'a' in scope}}
+  a = 2
+  // expected-error@+1{{cannot find 'b' in scope}}
+  b = 2
+
+  // expected-error@+1{{cannot find 'c' in scope}}
+  c = 2
+}

test/Utils/update-verify-tests/multiple-missing-errors-same-line.swift

@@ -0,0 +1,30 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+  a = 2;  b = 2; c = 2;
+}
+
+func bar() {
+  x = 2;  y = 2; z = 2;
+  // expected-error@-1{{cannot find 'x' in scope}}
+}
+//--- test.swift.expected
+func foo() {
+  // expected-error@+3{{cannot find 'c' in scope}}
+  // expected-error@+2{{cannot find 'b' in scope}}
+  // expected-error@+1{{cannot find 'a' in scope}}
+  a = 2;  b = 2; c = 2;
+}
+
+func bar() {
+  x = 2;  y = 2; z = 2;
+  // expected-error@-1{{cannot find 'x' in scope}}
+  // expected-error@-2{{cannot find 'y' in scope}}
+  // expected-error@-3{{cannot find 'z' in scope}}
+}

test/Utils/update-verify-tests/no-checks.swift

@@ -0,0 +1,16 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+  bar = 2
+}
+//--- test.swift.expected
+func foo() {
+  // expected-error@+1{{cannot find 'bar' in scope}}
+  bar = 2
+}

test/Utils/update-verify-tests/no-diags.swift

@@ -0,0 +1,16 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+  // expected-error@+1{{asdf}}
+  let _ = 2
+}
+//--- test.swift.expected
+func foo() {
+  let _ = 2
+}

test/Utils/update-verify-tests/non-default-prefix.swift

@@ -0,0 +1,24 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -verify-additional-prefix check- -typecheck %t/test.swift 2>&1 | %update-verify-tests --prefix check-
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift -verify-additional-prefix check- 
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+  a = 2
+  // expected-check-error{{foo}}
+  // expected-error{{bar}}
+
+  // expected-error@+1{{baz}}
+  b = 3
+}
+//--- test.swift.expected
+func foo() {
+  // expected-check-error@+1{{cannot find 'a' in scope}}
+  a = 2
+
+  // expected-error@+1{{cannot find 'b' in scope}}
+  b = 3
+}

test/Utils/update-verify-tests/update-same-line.swift

@@ -0,0 +1,17 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+  bar = 2     //   expected-error       {{asdf}}
+}
+
+//--- test.swift.expected
+func foo() {
+  bar = 2     //   expected-error       {{cannot find 'bar' in scope}}
+}
+

test/Utils/update-verify-tests/update-single-check.swift

@@ -0,0 +1,19 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+  // expected-error@+1{{AAA}}
+  bar = 2
+}
+
+//--- test.swift.expected
+func foo() {
+  // expected-error@+1{{cannot find 'bar' in scope}}
+  bar = 2
+}
+

test/Utils/update-verify-tests/wrong-category.swift

@@ -0,0 +1,19 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: not %target-swift-frontend-verify -typecheck %t/test.swift 2>&1 | %update-verify-tests
+// RUN: %target-swift-frontend-verify -typecheck %t/test.swift
+// RUN: %diff %t/test.swift %t/test.swift.expected
+
+//--- test.swift
+func foo() {
+  // expected-warning@+1{{cannot find 'bar' in scope}}
+  bar = 2
+}
+
+//--- test.swift.expected
+func foo() {
+  // expected-error@+1{{cannot find 'bar' in scope}}
+  bar = 2
+}
+

test/lit.cfg

@@ -766,6 +766,9 @@ config.substitutions.append( ('%validate-json', f"{config.python} -m json.tool")
 config.clang_include_dir = make_path(config.llvm_obj_root, 'include')
 config.substitutions.append( ('%clang-include-dir', config.clang_include_dir) )
 
+config.update_verify_tests = make_path(config.swift_utils, "update-verify-tests.py")
+config.substitutions.append( ('%update-verify-tests', '%s %s' % (config.python, config.update_verify_tests)) )
+
 config.swift_include_dir = make_path(config.swift_obj_root, 'include')
 config.substitutions.append( ('%swift-include-dir', config.swift_include_dir) )
 

utils/update-verify-tests.py

@@ -0,0 +1,44 @@
+import sys
+import argparse
+from update_verify_tests.core import check_expectations
+
+"""
+ Pipe output from swift-frontend's -verify into this script to have the test case updated to expect the actual diagnostic output.
+ When inserting new expected-* checks it will place them on the line before the location of the diagnostic, with an @+1,
+ or @+N for some N if there are multiple diagnostics emitted on the same line. If the current checks are using @-N for
+ this line, the new check will follow that convention also.
+ Existing checks will be left untouched as much as possible, including their location and whitespace content, to minimize
+ diffs. If inaccurate their count will be updated, or the check removed entirely.
+
+ Missing features:
+  - multiple prefixes on the same line (-verify-additional-prefix my-prefix -verify-additional-prefix my-other-prefix)
+  - multiple prefixes on separate RUN lines (RUN: -verify-additional-prefix my-prefix\nRUN: -verify-additional-prefix my-other-prefix)
+  - regexes matchers
+  - multiple checks targeting the same line are supported, but a line may only contain one check
+  - if multiple checks targeting the same line are failing the script is not guaranteed to produce a minimal diff
+  - remarks
+  - expansions
+  - columns
+  - fix-its
+  - doc files
+
+Example usage:
+  swift-frontend -verify [file] | python3 update-verify-tests.py
+  swift-frontend -verify -verify-additional-prefix check- [file] | python3 update-verify-tests.py --prefix check-
+"""
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--prefix", default="", help="The prefix passed to -verify"
+    )
+    args = parser.parse_args()
+    (ret_code, output) = check_expectations(sys.stdin.readlines(), args.prefix)
+    print(output)
+    sys.exit(ret_code)
+
+
+if __name__ == "__main__":
+    main()
+