[test] Warning about Swift features that do not exist (swiftlang#77542)

Sometimes features are removed from `Features.def`, but they are not
removed from the test files. The compiler ignores every feature that
does not exist. This leads to removed features still being tested, and
with the introduction of swiftlang#76740, `REQUIRED:`.

Modify the code that generates the active feature set for testing to
also generate the set of existing features, and pass this list of
existing features to the verifier script. If a feature is trying to be
activated and does not exist, the verifier will warn the user.

- `SwiftParser` feature was renamed `ParserASTGen`, but some tests were
  using both spellings. Remove the mentions of `SwiftParser` in the
  tests.
- `ImportObjcForwardDeclarations` was spelled with upper case `C` in
  a couple of tests. Fix it so the matching is easier.
- `TransferringArgsAndResults` was an experimental feature that was
  removed.
- Ignore the usage of inexisting feature in `swift-export-as.swift`
  because it seems to be what the test is actually testing.
- Ignore the test `availability_define.swift` because it tests the
  pseudo-feature `AvailabilityMacro=` which is not part of
  `Features.def`.

Author: drodriguez

test/ClangImporter/incomplete_objc_types_full_availability.swift

@@ -2,12 +2,12 @@
 // RUN: %target-clang %S/Inputs/custom-modules/IncompleteTypes/incomplete-type-library-1.m -c -o %t/incomplete-type-library-1.o
 // RUN: %target-clang %S/Inputs/custom-modules/IncompleteTypes/complete-types.m -c -o %t/complete-types.o
 
-// RUN: %target-build-swift -Xfrontend -enable-upcoming-feature -Xfrontend ImportObjCForwardDeclarations -Xfrontend -enable-objc-interop -I %S/Inputs/custom-modules/IncompleteTypes %s %t/incomplete-type-library-1.o %t/complete-types.o -Xlinker -framework -Xlinker Foundation -o %t/a.out
+// RUN: %target-build-swift -Xfrontend -enable-upcoming-feature -Xfrontend ImportObjcForwardDeclarations -Xfrontend -enable-objc-interop -I %S/Inputs/custom-modules/IncompleteTypes %s %t/incomplete-type-library-1.o %t/complete-types.o -Xlinker -framework -Xlinker Foundation -o %t/a.out
 // RUN: %target-run %t/a.out
 
 // REQUIRES: objc_interop
 // REQUIRES: executable_test
-// REQUIRES: swift_feature_ImportObjCForwardDeclarations
+// REQUIRES: swift_feature_ImportObjcForwardDeclarations
 
 import IncompleteTypeLibrary1
 import CompleteTypes

test/Concurrency/sending_continuation.swift

@@ -1,7 +1,6 @@
-// RUN: %target-swift-frontend -emit-sil -swift-version 6 -target %target-swift-5.1-abi-triple -enable-experimental-feature TransferringArgsAndResults -verify %s -o /dev/null -parse-as-library
+// RUN: %target-swift-frontend -emit-sil -swift-version 6 -target %target-swift-5.1-abi-triple -verify %s -o /dev/null -parse-as-library
 
 // REQUIRES: concurrency
-// REQUIRES: swift_feature_TransferringArgsAndResults
 
 ////////////////////////
 // MARK: Declarations //

test/Misc/verify-swift-feature-testing.test-sh

@@ -1,7 +1,8 @@
 #! /usr/bin/env python3
 # -*- python -*-
-# RUN: %{python} %s '%swift_src_root'
+# RUN: %{python} %s '%swift_src_root' %existing-swift-features
 
+import json
 import pathlib
 import re
 import subprocess
@@ -11,8 +12,14 @@ import sys
 # Tests that check for the behaviour of  experimental/upcoming features, so
 # they cannot automatically be checked.
 EXCEPTIONAL_FILES = [
+    # Tests for ParserValidation not being defined in no-asserts compilers
     pathlib.Path("test/Frontend/experimental-features-no-asserts.swift"),
+    # Tests for UnknownFeature not existing
     pathlib.Path("test/Frontend/upcoming_feature.swift"),
+    # Tests for ModuleInterfaceExportAs being ignored
+    pathlib.Path("test/ModuleInterface/swift-export-as.swift"),
+    # Uses the pseudo-feature AvailabilityMacro=
+    pathlib.Path("test/Sema/availability_define.swift"),
 ]
 
 FEATURE_USAGE_RE = re.compile(
@@ -28,16 +35,19 @@ def find_test_files_with_features_usage(swift_src_root):
     # mention `-enable-experimental-feature` or `-enable-upcoming-feature`.
     # Be careful of not using REQUIRES or RUN with a colon after them or Lit will
     # pick them up.
-    output = subprocess.check_output([
-        "grep",
-        "--extended-regexp",
-        "--recursive",
-        "-e",
-        "RUN[:].*-enable-(experimental|upcoming)-feature",
-        "--files-with-matches",
-        str(swift_src_root / "test"),
-        str(swift_src_root / "validation-test"),
-    ], text=True)
+    output = subprocess.check_output(
+        [
+            "grep",
+            "--extended-regexp",
+            "--recursive",
+            "-e",
+            "RUN[:].*-enable-(experimental|upcoming)-feature",
+            "--files-with-matches",
+            str(swift_src_root / "test"),
+            str(swift_src_root / "validation-test"),
+        ],
+        text=True,
+    )
     return output.splitlines()
 
 
@@ -46,91 +56,110 @@ def find_test_files_with_marker_usage(swift_src_root):
     # that mention `swift_feature_`.
     # Be careful of not using REQUIRES with a colon after them or Lit will
     # pick them up.
-    output = subprocess.check_output([
-        "grep",
-        "--extended-regexp",
-        "--recursive",
-        "-e",
-        "REQUIRES[:].*swift_feature_",
-        "--files-with-matches",
-        str(swift_src_root / "test"),
-        str(swift_src_root / "validation-test"),
-    ], text=True)
+    output = subprocess.check_output(
+        [
+            "grep",
+            "--extended-regexp",
+            "--recursive",
+            "-e",
+            "REQUIRES[:].*swift_feature_",
+            "--files-with-matches",
+            str(swift_src_root / "test"),
+            str(swift_src_root / "validation-test"),
+        ],
+        text=True,
+    )
     return output.splitlines()
 
 
 def find_run_lines(test_file):
-    output = subprocess.check_output([
-        "grep",
-        "--extended-regexp",
-        "--no-filename",
-        "-e",
-        "RUN[:]",
-        str(test_file),
-    ], text=True)
+    output = subprocess.check_output(
+        [
+            "grep",
+            "--extended-regexp",
+            "--no-filename",
+            "-e",
+            "RUN[:]",
+            str(test_file),
+        ],
+        text=True,
+    )
     return output.splitlines()
 
 
 def find_requires_lines(test_file):
-    output = subprocess.check_output([
-        "grep",
-        "--extended-regexp",
-        "--no-filename",
-        "-e",
-        "REQUIRES[:]",
-        str(test_file),
-    ], text=True)
+    output = subprocess.check_output(
+        [
+            "grep",
+            "--extended-regexp",
+            "--no-filename",
+            "-e",
+            "REQUIRES[:]",
+            str(test_file),
+        ],
+        text=True,
+    )
     return output.splitlines()
 
 
 def check_existing_requires(test_file, feature):
-    returncode = subprocess.call([
-        "grep",
-        "--extended-regexp",
-        "--quiet",
-        "-e",
-        "REQUIRES[:].*swift_feature_" + feature,
-        str(test_file),
-    ])
+    returncode = subprocess.call(
+        [
+            "grep",
+            "--extended-regexp",
+            "--quiet",
+            "-e",
+            "REQUIRES[:].*swift_feature_" + feature,
+            str(test_file),
+        ]
+    )
     return returncode != 0
 
 
 def check_existing_feature_usage(test_file, feature):
-    returncode = subprocess.call([
-        "grep",
-        "--extended-regexp",
-        "--quiet",
-        "-e",
-        (
-            "RUN[:].*-enable-(experimental|upcoming)-feature (-Xfrontend )?"
-            + re.escape(feature)
-        ),
-        str(test_file),
-    ])
+    returncode = subprocess.call(
+        [
+            "grep",
+            "--extended-regexp",
+            "--quiet",
+            "-e",
+            (
+                "RUN[:].*-enable-(experimental|upcoming)-feature (-Xfrontend )?"
+                + re.escape(feature)
+            ),
+            str(test_file),
+        ]
+    )
     return returncode != 0
 
 
 def check_existing_error_message_checks(test_file, feature):
-    returncode = subprocess.call([
-        "grep",
-        "--extended-regexp",
-        "--quiet",
-        "-e",
-        "requires '-enable-(experimental|upcoming)-feature " + feature + "'",
-        str(test_file),
-    ])
+    returncode = subprocess.call(
+        [
+            "grep",
+            "--extended-regexp",
+            "--quiet",
+            "-e",
+            "requires '-enable-(experimental|upcoming)-feature " + feature + "'",
+            str(test_file),
+        ]
+    )
     return returncode != 0
 
 
-def check_test_file_feature_usage(test_file):
+def check_test_file_feature_usage(test_file, existing_swift_features):
     run_lines = find_run_lines(test_file)
     features = set(
-        feature
-        for line in run_lines
-        for feature in FEATURE_USAGE_RE.findall(line)
+        feature for line in run_lines for feature in FEATURE_USAGE_RE.findall(line)
     )
     num_failures = 0
     for feature in features:
+        # First, check this is a valid feature
+        if feature not in existing_swift_features:
+            print("error: {}: Unknown feature: {}".format(str(test_file), feature))
+            num_failures += 1
+            continue
+
         # No warning if the necessary `REQUIRES` is already there
         if not check_existing_requires(test_file, feature):
             continue
@@ -142,9 +171,7 @@ def check_test_file_feature_usage(test_file):
         # For everything else, print a warning and add to the invalid exit code
         print(
             "error: {}: Missing '{}: swift_feature_{}'".format(
-                str(test_file),
-                "REQUIRES",
-                feature
+                str(test_file), "REQUIRES", feature
             )
         )
         num_failures += 1
@@ -167,20 +194,20 @@ def check_test_file_marker_usage(test_file):
         # For everything else, print a warning and add to the invalid exit code
         print(
             "error: {}: Missing '-enable-experimental/upcoming-feature: {}'".format(
-                str(test_file),
-                feature
+                str(test_file), feature
             )
         )
         num_failures += 1
     return num_failures == 0
 
 
 def main():
-    if len(sys.argv) < 2:
-        print('Invalid number of arguments.')
+    if len(sys.argv) < 3:
+        print("Invalid number of arguments.")
         sys.exit(1)
 
     swift_src_root = pathlib.Path(sys.argv[1])
+    existing_swift_features = set(json.loads(sys.argv[2]))
 
     num_failures = 0
     test_files_with_features_usage = find_test_files_with_features_usage(swift_src_root)
@@ -190,7 +217,7 @@ def main():
         if test_file.relative_to(swift_src_root) in EXCEPTIONAL_FILES:
             continue
 
-        if not check_test_file_feature_usage(test_file):
+        if not check_test_file_feature_usage(test_file, existing_swift_features):
             num_failures += 1
 
     test_files_with_marker_usage = find_test_files_with_marker_usage(swift_src_root)
@@ -208,5 +235,5 @@ def main():
         sys.exit(1)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

test/Sema/availability_define.swift

@@ -15,7 +15,6 @@
 // RUN:   -enable-experimental-feature AvailabilityMacro="_myProject 2.5:macOS 52.5"
 
 // REQUIRES: OS=macosx
-// REQUIRES: swift_feature_AvailabilityMacro
 
 @available(_iOS53Aligned, *)
 public func onMacOS50() {}

test/lit.cfg

@@ -18,15 +18,16 @@
 #
 # -----------------------------------------------------------------------------
 
+import glob
+import json
 import os
 import platform
 import re
 import shlex
 import shutil
+import socket
 import subprocess
 import sys
-import socket
-import glob
 
 import lit
 import lit.formats
@@ -790,6 +791,8 @@ config.substitutions.append( ('%clang',
                               "%r %s" %
                                 (config.clang, clang_mcp_opt)) )
 
+config.substitutions.append(('%existing-swift-features', shlex.quote(json.dumps(list(config.existing_swift_features)))))
+
 ###
 
 def disallow(execName):

test/lit.swift-features.cfg.inc

@@ -22,12 +22,14 @@
  * statements).
  */
 
+config.existing_swift_features = set()
 def language_feature(feature_name, enabled):
+    config.existing_swift_features.add(feature_name)
     if enabled or "asserts" in config.available_features:
-        config.available_features.add(feature_name)
+        config.available_features.add("swift_feature_" + feature_name)
 
-#define UPCOMING_FEATURE(FeatureName, SENumber, Version) language_feature("swift_feature_" # FeatureName, True)
-#define EXPERIMENTAL_FEATURE(FeatureName, AvailableInProd) language_feature("swift_feature_" # FeatureName, #AvailableInProd == "true")
-#define LANGUAGE_FEATURE(FeatureName, SENumber, Description) language_feature("swift_feature_" # FeatureName, True)
+#define UPCOMING_FEATURE(FeatureName, SENumber, Version) language_feature(#FeatureName, True)
+#define EXPERIMENTAL_FEATURE(FeatureName, AvailableInProd) language_feature(#FeatureName, #AvailableInProd == "true")
+#define LANGUAGE_FEATURE(FeatureName, SENumber, Description) language_feature(#FeatureName, True)
 
 #include <swift/Basic/Features.def>