Skip to content

Commit

Permalink
Use custom script for test discovery
Browse files Browse the repository at this point in the history
Standard test discovery is not available in Swift 5.0, but we want to
support this version of Swift.

Besides, standard test discovery doesn't work very well with CircleCI:
https://bugs.swift.org/browse/SR-10783
  • Loading branch information
broadwaylamb committed Oct 29, 2020
1 parent 9100cca commit 8786d08
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 25 deletions.
35 changes: 10 additions & 25 deletions .circleci/config.yml
Expand Up @@ -65,49 +65,34 @@ ubuntu_tests_steps: &ubuntu_tests_steps
command: |
apt update -y
apt upgrade -y
apt install -y curl
apt install -y curl python3.8
- run:
name: "Generating LinuxMain.swift"
command: python3.8 utils/discover_tests.py
- run:
name: Building and running tests in debug mode with coverage
command: | # We need to run the test command twice because of https://bugs.swift.org/browse/SR-10783
make test-debug \
SWIFT_TEST_FLAGS="--enable-test-discovery \
--enable-index-store \
--enable-code-coverage \
--build-path .build-test-debug" \
> /dev/null 2>&1 \
|| true
command: |
make test-debug \
SWIFT_BUILD_FLAGS="-Xswiftc -warnings-as-errors" \
SWIFT_TEST_FLAGS="--enable-test-discovery \
--enable-index-store \
--enable-code-coverage \
SWIFT_TEST_FLAGS="--enable-code-coverage \
--disable-index-store \
--build-path .build-test-debug"
llvm-cov show \
-instr-profile=.build-test-debug/debug/codecov/default.profdata \
.build-test-debug/debug/OpenCombinePackageTests.xctest \
> coverage.txt
- run:
name: Building and running tests in debug mode with TSan
command: | # We need to run the test command twice because of https://bugs.swift.org/browse/SR-10783
make test-debug-sanitize-thread \
SWIFT_BUILD_FLAGS="-Xswiftc -warnings-as-errors" \
SWIFT_TEST_FLAGS="--enable-test-discovery \
--enable-index-store \
--build-path .build-test-debug-sanitize-thread" \
> /dev/null 2>&1 \
|| true
command: |
make test-debug-sanitize-thread \
SWIFT_TEST_FLAGS="--enable-test-discovery \
--enable-index-store \
SWIFT_TEST_FLAGS="--disable-index-store \
--build-path .build-test-debug-sanitize-thread" \
- run:
name: Building and running tests in release mode
command: |
make test-release \
SWIFT_BUILD_FLAGS="-Xswiftc -warnings-as-errors" \
SWIFT_TEST_FLAGS="--enable-test-discovery \
--enable-index-store \
--build-path .build-test-release"
SWIFT_TEST_FLAGS="--build-path .build-test-release"
- run:
name: Uploading code coverage
command: |
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Expand Up @@ -3,6 +3,7 @@
/Packages
/*.xcodeproj
/.swiftpm
Tests/LinuxMain.swift

# Created by https://www.gitignore.io/api/Xcode
# Edit at https://www.gitignore.io/?templates=Xcode
Expand Down
67 changes: 67 additions & 0 deletions utils/discover_tests.py
@@ -0,0 +1,67 @@
"""
Parse test files and generate LinuxMain.swift.
This script uses simple regular expressions to extract test names from test files.
We don't use --enable-test-discovery because:
* it doesn't work with Swift 5.0 that we still support
* there are other problems with it: https://bugs.swift.org/browse/SR-10783
"""
import sys

if sys.version_info < (3, 6, 0):
exit("This script only works with Python 3.6 and newer.")

import re
from pathlib import Path
from argparse import ArgumentParser

TEST_METHOD_PATTERN = \
re.compile(r"func\s*(test\w+)\s*\(\s*\)\s*(?:throws\s*)?{")
TEST_DISCOVERY_CONDITION_PATTERN = \
re.compile(r"^#if (.+)\s*\/\/\s*TEST_DISCOVERY_CONDITION\s*$", flags=re.MULTILINE)

def extract_test_names(test_file):
contents = test_file.read_text()
test_names = [match[1] for match in TEST_METHOD_PATTERN.finditer(contents)]
condition = TEST_DISCOVERY_CONDITION_PATTERN.search(contents)
return (test_names, condition[1] if condition else None)

def generate_linuxmain(workdir):
workdir = Path(workdir)
tests_dir = workdir / "Tests"
test_files = (tests_dir / "OpenCombineTests").glob("*/*Tests.swift")

with (tests_dir / "LinuxMain.swift").open(mode="w") as linuxmain:
linuxmain.write("""\
import XCTest
@testable import OpenCombineTests
var tests = [XCTestCaseEntry]()
""")
for test_file in test_files:
(test_names, condition) = extract_test_names(test_file)
if not test_names:
continue
if condition:
linuxmain.write(f"#if {condition}\n")
linuxmain.write(f"let allTests_{test_file.stem} = [\n")
for test_name in test_names:
linuxmain.write(f" (\"{test_name}\", {test_file.stem}.{test_name}),\n")
linuxmain.write("]\n")
linuxmain.write(f"tests.append(testCase(allTests_{test_file.stem}))\n")
if condition:
linuxmain.write("#endif\n")
linuxmain.write("\n")

linuxmain.write("XCTMain(tests)")


if __name__ == "__main__":
parser = ArgumentParser(description="Generates LinuxMain.swift for OpenCombine tests")
parser.add_argument("workdir", type=Path, nargs="?",
help="The root directory of the OpenCombine Swift package")
args = parser.parse_args()
workdir = args.workdir or Path.cwd()
generate_linuxmain(workdir)

0 comments on commit 8786d08

Please sign in to comment.